VPP Python language binding - plugin support

- Moved Python generator tool to tools directory
- Added build-vpp-api Makefile target
- Generator now only creates a Python representation of the .api
  the rest of the framework is in the vpp_papi script
- Each plugin has its own namespace.
- Plugin Python files are installed in vpp_papi_plugins for easy
  use inside the build tree.

Change-Id: I272c83bb7e5d5e416bdbd8a790a3cc35c5a04e38
Signed-off-by: Ole Troan <ot@cisco.com>

23 files changed, 757 insertions, 157 deletions

diff --git a/.gitignore b/.gitignore
index 425261836aa..8cbf1e61e0e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,7 +13,6 @@
 /build-root/*.rpm
 /build-root/*.changes
 /build-config.mk
-/vpp-api/python/vpp_papi/vpp_papi.py
 /dpdk/*.tar.gz
 /dpdk/*.tar.xz
 /path_setup
@@ -51,7 +50,7 @@ test-driver
 .settings
 # stop autotools ignore
 
-# OSX and some IDE 
+# OSX and some IDE
 .DS_Store
 .idea/
 .project
@@ -63,6 +62,7 @@ test-driver
 GPATH
 GRTAGS
 GTAGS
+TAGS
 
 # Generated documentation
 /build-root/docs
diff --git a/Makefile b/Makefile
index 4e3d65bcd02..39930651289 100644
--- a/Makefile
+++ b/Makefile
@@ -54,7 +54,7 @@ endif
 
 .PHONY: help bootstrap wipe wipe-release build build-release rebuild rebuild-release
 .PHONY: run run-release debug debug-release build-vat run-vat pkg-deb pkg-rpm
-.PHONY: ctags cscope doxygen wipe-doxygen plugins plugins-release
+.PHONY: ctags cscope doxygen wipe-doxygen plugins plugins-release build-vpp-api
 
 help:
 	@echo "Make Targets:"
@@ -73,6 +73,7 @@ help:
 	@echo " debug               - run debug binary with debugger"
 	@echo " debug-release       - run release binary with debugger"
 	@echo " build-vat           - build vpp-api-test tool"
+	@echo " build-vpp-api       - build vpp-api"
 	@echo " run-vat             - run vpp-api-test tool"
 	@echo " pkg-deb             - build DEB packages"
 	@echo " pkg-rpm             - build RPM packages"
@@ -172,6 +173,9 @@ plugins: $(BR)/.bootstrap.ok
 plugins-release: $(BR)/.bootstrap.ok
 	$(call make,$(PLATFORM),plugins-install)
 
+build-vpp-api: $(BR)/.bootstrap.ok
+	$(call make,$(PLATFORM)_debug,vpp-api-install)
+
 STARTUP_DIR ?= $(PWD)
 ifeq ("$(wildcard $(STARTUP_CONF))","")
 define run
diff --git a/build-data/platforms.mk b/build-data/platforms.mk
index cd65f67c057..36cfc878b37 100644
--- a/build-data/platforms.mk
+++ b/build-data/platforms.mk
@@ -56,14 +56,16 @@ install-deb: $(patsubst %,%-find-source,$(ROOT_PACKAGES))
 	   >> deb/debian/vpp.install ;					\
 									\
 	: dev package needs a couple of additions ;			\
-    echo ../build-tool-native/vppapigen/vppapigen /usr/bin		\
-       >> deb/debian/vpp-dev.install ;				\
-    echo ../../vpp-api/java/jvpp/gen/jvpp_gen.py /usr/bin    \
-       >> deb/debian/vpp-dev.install ;				 \
-    for i in $$(ls ../vpp-api/java/jvpp/gen/jvppgen/*.py); do \
-    	echo ../$${i} /usr/lib/python2.7/dist-packages/jvppgen \
-    	   >> deb/debian/vpp-dev.install; \
-   	done; \
+	echo ../build-tool-native/vppapigen/vppapigen /usr/bin		\
+	   >> deb/debian/vpp-dev.install ;				\
+	echo ../../vppapigen/pyvppapigen.py /usr/bin			\
+           >> deb/debian/vpp-dev.install ;				\
+	echo ../../vpp-api/java/jvpp/gen/jvpp_gen.py /usr/bin		\
+	   >> deb/debian/vpp-dev.install ;				\
+	for i in $$(ls ../vpp-api/java/jvpp/gen/jvppgen/*.py); do	\
+	   echo ../$${i} /usr/lib/python2.7/dist-packages/jvppgen	\
+	       >> deb/debian/vpp-dev.install;				\
+	done;								\
 									\
 	: generate changelog;						\
 	./scripts/generate-deb-changelog 				\
diff --git a/plugins/ioam-plugin/Makefile.am b/plugins/ioam-plugin/Makefile.am
index 2ea29e006d9..68e792482a9 100644
--- a/plugins/ioam-plugin/Makefile.am
+++ b/plugins/ioam-plugin/Makefile.am
@@ -28,7 +28,7 @@ ioam_pot_plugin_la_SOURCES =			\
 	ioam/lib-pot/pot_api.c
 
 BUILT_SOURCES =					\
-	ioam/lib-pot/pot.api.h
+	ioam/lib-pot/pot.api.h ioam/lib-pot/pot.py
 
 SUFFIXES = .api.h .api
 
@@ -37,6 +37,15 @@ SUFFIXES = .api.h .api
 	$(CC) $(CPPFLAGS) -E -P -C -x c $^ \
 	| vppapigen --input - --output $@ --show-name $@
 
+%.py: %.api
+	$(info Creating Python binding for $@)
+	$(CC) $(CPPFLAGS) -E -P -C -x c $<				\
+	| vppapigen --input - --python -				\
+	| pyvppapigen.py --input - > $@
+
+pyapidir = ${prefix}/vpp_papi_plugins
+pyapi_DATA = ioam/lib-pot/pot.py
+
 noinst_HEADERS =				\
   ioam/lib-pot/pot_all_api_h.h			\
   ioam/lib-pot/pot_msg_enum.h			\
diff --git a/plugins/snat-plugin/Makefile.am b/plugins/snat-plugin/Makefile.am
index 0fe694cf7bd..91fec414363 100644
--- a/plugins/snat-plugin/Makefile.am
+++ b/plugins/snat-plugin/Makefile.am
@@ -28,7 +28,7 @@ snat_plugin_la_SOURCES = snat/snat.c		\
         snat/out2in.c				\
 	snat/snat_plugin.api.h
 
-BUILT_SOURCES = snat/snat.api.h
+BUILT_SOURCES = snat/snat.api.h snat/snat.py
 
 SUFFIXES = .api.h .api
 
@@ -37,6 +37,15 @@ SUFFIXES = .api.h .api
 	$(CC) $(CPPFLAGS) -E -P -C -x c $^ \
 	| vppapigen --input - --output $@ --show-name $@
 
+%.py: %.api
+	$(info Creating Python binding for $@)
+	$(CC) $(CPPFLAGS) -E -P -C -x c $<				\
+	| vppapigen --input - --python -				\
+	| pyvppapigen.py --input - > $@
+
+pyapidir = ${prefix}/vpp_papi_plugins
+pyapi_DATA = snat/snat.py
+
 noinst_HEADERS =			\
   snat/snat_all_api_h.h			\
   snat/snat_msg_enum.h			\
diff --git a/vlib-api/Makefile.am b/vlib-api/Makefile.am
index 4b0129a3a5f..5bc00e7479b 100644
--- a/vlib-api/Makefile.am
+++ b/vlib-api/Makefile.am
@@ -75,3 +75,7 @@ SUFFIXES = .api.h .api
 	mkdir -p `dirname $@` ;					\
 	$(CC) $(CPPFLAGS) -E -P -C -x c $^			\
 	| vppapigen --input - --output $@ --show-name $@
+
+# install the API definition, so we can produce java bindings, etc.
+apidir = $(prefix)/vlibmemory
+api_DATA = vlibmemory/memclnt.api
diff --git a/vpp-api/python/Makefile.am b/vpp-api/python/Makefile.am
index 4d2d221d231..eb589335b05 100644
--- a/vpp-api/python/Makefile.am
+++ b/vpp-api/python/Makefile.am
@@ -13,33 +13,50 @@
 
 AUTOMAKE_OPTIONS = foreign subdir-objects
 ACLOCAL_AMFLAGS = -I m4
-AM_CFLAGS = -Wall 
+AM_CFLAGS = -Wall
 
 BUILT_SOURCES =
-bin_PROGRAMS = 
-CLEANFILES = 
-lib_LTLIBRARIES = 
+bin_PROGRAMS =
+CLEANFILES =
+lib_LTLIBRARIES =
 noinst_PROGRAMS = test_pneum
 nobase_include_HEADERS = pneum/pneum.h
 
 #
-# Python binding
+# Python / C extension
 #
+lib_LTLIBRARIES += vpp_api.la
+vpp_api_la_SOURCES = pneum/pneum.c vpp_papi/pneum_wrap.c
+vpp_api_la_LIBADD = -lvlibmemoryclient -lvlibapi -lsvm -lvppinfra -lpthread -lm -lrt
+vpp_api_la_LDFLAGS = -module $(shell python-config --ldflags)
+vpp_api_la_CPPFLAGS = $(shell python-config --includes)
+
+# Kept around for setuptools based install.
 lib_LTLIBRARIES += libpneum.la
-libpneum_la_SOURCES = pneum/pneum.c
+libpneum_la_SOURCES = pneum/pneum.c setup.py
 libpneum_la_LIBADD = -lvlibmemoryclient -lvlibapi -lsvm -lvppinfra -lpthread -lm -lrt
 libpneum_la_LDFLAGS = -module
 libpneum_la_CPPFLAGS =
 
-BUILT_SOURCES += vpp_papi.py
+#
+# Core VPP API
+#
+BUILT_SOURCES +=					\
+	$(prefix)/../vpp/vpp-api/vpe.py			\
+	$(prefix)/../vlib-api/vlibmemory/memclnt.py
 
-vpp_papi.py: $(prefix)/../vpp/vpp-api/vpe.api pneum/api-gen.py
-	@echo "  PYTHON API";                                            	  \
-	$(CC) $(CPPFLAGS) -E -P -C -x c $<			                  \
-	| vppapigen --input - --python defs_$@;                                   \
-	echo "#include <vpp-api/vpe_msg_enum.h>"                                      \
-	| $(CC) $(CPPFLAGS) -E -P -x c - | grep VL_API                            \
-	| @srcdir@/pneum/api-gen.py -i defs_$@ > @srcdir@/vpp_papi/$@
+%.py: %.api
+	$(info Creating Python binding for $@)
+	$(CC) $(CPPFLAGS) -E -P -C -x c $<	\
+	| vppapigen --input - --python -	\
+	| pyvppapigen.py --input - > $@
+
+#
+# TODO: Support both Python 2 and 3.
+install-exec-local:
+	cd $(srcdir); \
+	mkdir -p $(prefix)/lib/python2.7/site-packages; \
+	PYTHONUSERBASE=$(prefix) python setup.py install --user
 
 #
 # Test client
diff --git a/vpp-api/python/pneum/pneum.c b/vpp-api/python/pneum/pneum.c
index ac518493163..2637d43f5d9 100644
--- a/vpp-api/python/pneum/pneum.c
+++ b/vpp-api/python/pneum/pneum.c
@@ -10,7 +10,7 @@
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
- * limitations under the License. 
+ * limitations under the License.
  */
 #include <stdio.h>
 #include <stdlib.h>
@@ -36,11 +36,11 @@
 #include "pneum.h"
 
 #define vl_typedefs             /* define message structures */
-#include <vpp-api/vpe_all_api_h.h> 
+#include <vpp-api/vpe_all_api_h.h>
 #undef vl_typedefs
 
 #define vl_endianfun             /* define message structures */
-#include <vpp-api/vpe_all_api_h.h> 
+#include <vpp-api/vpe_all_api_h.h>
 #undef vl_endianfun
 
 typedef struct {
@@ -54,12 +54,12 @@ pneum_main_t pneum_main;
 
 extern int wrap_pneum_callback(char *data, int len);
 
-/* 
+/*
  * Satisfy external references when -lvlib is not available.
  */
 void vlib_cli_output (struct vlib_main_t * vm, char * fmt, ...)
 {
-  clib_warning ("vlib_cli_output callled...");
+  clib_warning ("vlib_cli_output called...");
 }
 
 #define vl_api_version(n,v) static u32 vpe_api_version = v;
@@ -89,7 +89,7 @@ pneum_api_handler (void *msg)
   int l = ntohl(msgbuf->data_len);
   if (l == 0)
     clib_warning("Message ID %d has wrong length: %d\n", id, l);
-    
+
   /* Call Python callback */
   (void)wrap_pneum_callback(msg, l);
   vl_msg_api_free(msg);
@@ -121,12 +121,6 @@ pneum_connect (char *name)
   int rv = 0;
   pneum_main_t *pm = &pneum_main;
 
-  /*
-   * Bail out now if we're not running as root
-   */
-  if (geteuid() != 0)
-    return (-1);
-
   if ((rv = vl_client_api_map("/vpe-api"))) {
     clib_warning ("vl_client_api map rv %d", rv);
     return rv;
@@ -199,7 +193,7 @@ pneum_read (char **p, int *l)
     *p = (char *)msg;
   } else {
     printf("Read failed with %d\n", rv);
-  }  
+  }
   return (rv);
 }
 
diff --git a/vpp-api/python/pneum/pneum.h b/vpp-api/python/pneum/pneum.h
index b99cbd4e3ec..75fccf84bcf 100644
--- a/vpp-api/python/pneum/pneum.h
+++ b/vpp-api/python/pneum/pneum.h
@@ -12,10 +12,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#ifndef __included_pneum_h__
-#define __included_pneum_h__
+#ifndef included_pneum_h
+#define included_pneum_h
 
-unsigned int vpe_client_index(void);
 int pneum_connect(char *name);
 int pneum_disconnect(void);
 int pneum_read(char **data, int *l);
diff --git a/vpp-api/python/setup.cfg b/vpp-api/python/setup.cfg
new file mode 100644
index 00000000000..5e19e8c5e74
--- /dev/null
+++ b/vpp-api/python/setup.cfg
@@ -0,0 +1,7 @@
+[bdist_wheel]
+# This flag says that the code is written to work on both Python 2 and Python
+# 3. If at all possible, it is good practice to do this. If you cannot, you
+# will need to generate wheels for each Python version that you support.
+universal=0
+
+
diff --git a/vpp-api/python/setup.py b/vpp-api/python/setup.py
index d890ba709dc..e369a0cb479 100644
--- a/vpp-api/python/setup.py
+++ b/vpp-api/python/setup.py
@@ -1,21 +1,16 @@
-from distutils.core import setup, Extension
-
-module1 = Extension('vpp_api',
-                    define_macros = [('MAJOR_VERSION', '1'),
-                                     ('MINOR_VERSION', '0')],
-                    include_dirs = ['pneum'],
-                    libraries = ['pneum'],
-                    library_dirs = ['../../build-root/install-vpp_debug-native/vpp-api/lib64'],
-                    sources = ['vpp_papi/pneum_wrap.c'])
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
 
 setup (name = 'vpp_papi',
-       version = '1.0',
+       version = '1.1',
        description = 'VPP Python binding',
        author = 'Ole Troan',
        author_email = 'ot@cisco.com',
        #url = 'https://docs.python.org/extending/building',
+       test_suite = 'tests',
        packages=['vpp_papi'],
        long_description = '''
 VPP Python language binding.
-''',
-       ext_modules = [module1])
+''',)
diff --git a/vpp-api/python/tests/test_base.py b/vpp-api/python/tests/test_base.py
new file mode 100644
index 00000000000..8ff5dd4782c
--- /dev/null
+++ b/vpp-api/python/tests/test_base.py
@@ -0,0 +1,7 @@
+# Manipulate sys.path to allow tests be run inside the build environment.
+import os, sys, glob
+scriptdir = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.dirname(glob.glob(scriptdir+'/../../../build-root/install*/vpp-api/lib64/vpp_api.so')[0]))
+sys.path.append(os.path.dirname(glob.glob(scriptdir+'/../../../build-root/install*/vlib-api/vlibmemory/memclnt.py')[0]))
+sys.path.append(os.path.dirname(glob.glob(scriptdir+'/../../../build-root/install*/vpp/vpp-api/vpe.py')[0]))
+sys.path.append(glob.glob(scriptdir+'/../../../build-root/install*/plugins/vpp_papi_plugins')[0])
diff --git a/vpp-api/python/tests/test_modules.py b/vpp-api/python/tests/test_modules.py
new file mode 100755
index 00000000000..f3066b29395
--- /dev/null
+++ b/vpp-api/python/tests/test_modules.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import unittest
+import test_base
+import vpp_papi
+import pot, snat
+print('Plugins:')
+vpp_papi.plugin_show()
+r = vpp_papi.connect('ole')
+
+r = vpp_papi.show_version()
+print('R:', r)
+
+r = snat.snat_interface_add_del_feature(1, 1, 1)
+print('R:', r)
+
+vpp_papi.disconnect()
diff --git a/vpp-api/python/tests/test_papi.py b/vpp-api/python/tests/test_papi.py
index bede7171519..ab90eeaa45c 100755
--- a/vpp-api/python/tests/test_papi.py
+++ b/vpp-api/python/tests/test_papi.py
@@ -1,102 +1,104 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
-import vpp_papi
+from __future__ import print_function
 import unittest, sys, time, threading, struct, logging
+import test_base
+import vpp_papi
 from ipaddress import *
 
 papi_event = threading.Event()
+print(vpp_papi.VL_API_SW_INTERFACE_SET_FLAGS)
 def papi_event_handler(result):
-    if result.vl_msg_id == vpp_papi.VL_API_SW_INTERFACE_SET_FLAGS:
-        papi_event.set()
+    if result.vl_msg_id == vpp_papi.vpe.VL_API_SW_INTERFACE_SET_FLAGS:
         return
-    if result.vl_msg_id == vpp_papi.VL_API_VNET_INTERFACE_COUNTERS:
-        format = '>' + str(int(len(result.data) / 8)) + 'Q'
-        counters = struct.unpack(format, result.data)
-        print('Counters:', counters)
+    if result.vl_msg_id == vpp_papi.vpe.VL_API_VNET_INTERFACE_COUNTERS:
+        print('Interface counters', result)
         return
-    if result.vl_msg_id == vpp_papi.VL_API_VNET_IP6_FIB_COUNTERS:
-        print('IP6 FIB Counters:', result.count, len(result.c), len(result))
-        i = 0
-        # FIB counters allocate a large (1000 bytes) block so message length does not match reality
-        for c in struct.iter_unpack('>16sBQQ', result.c):
-            # In Python 3.5 we can use a tuple for prefix, length
-            print(str(IPv6Address(c[0])) + '/' + str(c[1]), str(c[2]), str(c[3]))
-            i += 1
-            if i >= result.count:
-                break
+    if result.vl_msg_id == vpp_papi.vpe.VL_API_VNET_IP6_FIB_COUNTERS:
+        print('IPv6 FIB counters', result)
+        papi_event.set()
         return
 
     print('Unknown message id:', result.vl_msg_id)
 
+import glob, subprocess
 class TestPAPI(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        #
+        # Start main VPP process
+        cls.vpp_bin = glob.glob(test_base.scriptdir+'/../../../build-root/install-vpp*-native/vpp/bin/vpp')[0]
+        print("VPP BIN:", cls.vpp_bin)
+        cls.vpp = subprocess.Popen([cls.vpp_bin, "unix", "nodaemon"], stderr=subprocess.PIPE)
+        print('Started VPP')
+        # For some reason unless we let VPP start up the API cannot connect.
+        time.sleep(0.3)
+    @classmethod
+    def tearDownClass(cls):
+        cls.vpp.terminate()
 
     def setUp(self):
+        print("Connecting API")
         r = vpp_papi.connect("test_papi")
         self.assertEqual(r, 0)
 
     def tearDown(self):
         r = vpp_papi.disconnect()
         self.assertEqual(r, 0)
-        
+
+    #
+    # The tests themselves
+    #
+
+    #
+    # Basic request / reply
+    #
     def test_show_version(self):
         t = vpp_papi.show_version()
+        print('T', t);
         program = t.program.decode().rstrip('\x00')
         self.assertEqual('vpe', program)
 
     #
-    # Add a few MAP domains, then dump them later
+    # Details / Dump
     #
-    def test_map(self):
-        t = vpp_papi.map_summary_stats()
-        print(t)
-        ip6 = IPv6Address(u'2001:db8::1').packed
-        ip4 = IPv4Address(u'10.0.0.0').packed
-        ip6_src = IPv6Address(u'2001:db9::1').packed
-        t = vpp_papi.map_add_domain(ip6, ip4, ip6_src, 32, 24, 128, 0, 0, 6, 0, 0)
-        print(t)
-        self.assertEqual(t.retval, 0)
+    def test_details_dump(self):
+        t = vpp_papi.sw_interface_dump(0, b'')
+        print('Dump/details T', t)
 
-        ip4 = IPv4Address(u'10.0.1.0').packed
-        t = vpp_papi.map_add_domain(ip6, ip4, ip6_src, 32, 24, 128, 0, 0, 6, 0, 0)
+    #
+    # Arrays
+    #
+    def test_arrays(self):
+        t = vpp_papi.vnet_get_summary_stats()
+        print('Summary stats', t)
+        print('Packets:', t.total_pkts[0])
+        print('Packets:', t.total_pkts[1])
+    #
+    # Variable sized arrays and counters
+    #
+    #@unittest.skip("stats")
+    def test_want_stats(self):
+        pid = 123
+        vpp_papi.register_event_callback(papi_event_handler)
+        papi_event.clear()
+
+        # Need to configure IPv6 to get som IPv6 FIB stats
+        t = vpp_papi.create_loopback('')
         print(t)
         self.assertEqual(t.retval, 0)
 
-        t = vpp_papi.map_summary_stats()
+        ifindex = t.sw_if_index
+        addr = str(IPv6Address('1::1').packed)
+        t = vpp_papi.sw_interface_add_del_address(ifindex, 1, 1, 0, 16, addr)
         print(t)
-        self.assertEqual(t.total_bindings, 2)
-
-        t = vpp_papi.map_domain_dump()
-        print (t)
-        self.assertEqual(len(t), 2)
-
-    def test_sw_interface_dump(self):
-        #
-        # Dump interfaces
-        #
-        t = vpp_papi.sw_interface_dump(0, b'ignored')
-        for interface in t:
-            if interface.vl_msg_id == vpp_papi.VL_API_SW_INTERFACE_DETAILS:
-                print(interface.interface_name.decode())
+        self.assertEqual(t.retval, 0)
 
-    def test_want_interface_events(self):
-        pid = 123
-        vpp_papi.register_event_callback(papi_event_handler)
-        papi_event.clear()
-        t = vpp_papi.want_interface_events(True, pid)
-        print (t)
-        print('Setting interface up')
-        t = vpp_papi.sw_interface_set_flags(0, 1, 1, 0)
-        print (t)
-        self.assertEqual(papi_event.wait(5), True)
-        t = vpp_papi.sw_interface_set_flags(0, 0, 0, 0)
-        print (t)
-        self.assertEqual(papi_event.wait(5), True)
+        # Check if interface is up
+        # XXX: Add new API to query interface state based on ifindex, instead of dump all.
+        t = vpp_papi.sw_interface_set_flags(ifindex, 1, 1, 0)
+        self.assertEqual(t.retval, 0)
 
-    @unittest.skip("not quite ready yet")
-    def test_want_stats(self):
-        pid = 123
-        vpp_papi.register_event_callback(papi_event_handler)
-        papi_event.clear()
         t = vpp_papi.want_stats(True, pid)
 
         print (t)
@@ -104,33 +106,17 @@ class TestPAPI(unittest.TestCase):
         #
         # Wait for some stats
         #
-        self.assertEqual(papi_event.wait(30), True)
+        self.assertEqual(papi_event.wait(15), True)
         t = vpp_papi.want_stats(False, pid)
         print (t)
 
-    def test_tap(self):
-        pid = 123
-        vpp_papi.register_event_callback(papi_event_handler)
-        papi_event.clear()
-        t = vpp_papi.want_stats(True, pid)
-
-        print (t)
-
-        t = vpp_papi.tap_connect(1, b'tap', b'foo', 1, 0)
-        print (t)
-        self.assertEqual(t.retval, 0)
-        swifindex = t.sw_if_index
-
-        t = vpp_papi.sw_interface_set_flags(swifindex, 1, 1, 0)
-        print (t)
-        self.assertEqual(t.retval, 0)        
-
-        ip6 = IPv6Address(u'2001:db8::1').packed
-        t = vpp_papi.sw_interface_add_del_address(swifindex, 1, 1, 0, 16, ip6)
-        print (t)
-        time.sleep(40)
 
+    #
+    # Plugins?
+    #
 
 if __name__ == '__main__':
     #logging.basicConfig(level=logging.DEBUG)
     unittest.main()
+def test_papi():
+    print('test')
diff --git a/vpp-api/python/vpp_papi/__init__.py b/vpp-api/python/vpp_papi/__init__.py
index 8be644d7cb7..19d78a3a05d 100644
--- a/vpp-api/python/vpp_papi/__init__.py
+++ b/vpp-api/python/vpp_papi/__init__.py
@@ -1,2 +1,4 @@
 __import__('pkg_resources').declare_namespace(__name__)
-from .vpp_papi import *
+from . vpp_papi import *
+
+
diff --git a/vpp-api/python/vpp_papi/pneum_wrap.c b/vpp-api/python/vpp_papi/pneum_wrap.c
index d1795aa13a7..7a5119746be 100644
--- a/vpp-api/python/vpp_papi/pneum_wrap.c
+++ b/vpp-api/python/vpp_papi/pneum_wrap.c
@@ -1,5 +1,20 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 #include <Python.h>
-#include "pneum.h"
+#include "../pneum/pneum.h"
 
 static PyObject *pneum_callback = NULL;
 
@@ -35,7 +50,7 @@ wrap_connect (PyObject *self, PyObject *args)
 
   if (!PyArg_ParseTuple(args, "sO:set_callback", &name, &temp))
     return (NULL);
-  
+
   if (!PyCallable_Check(temp)) {
     PyErr_SetString(PyExc_TypeError, "parameter must be callable");
     return NULL;
@@ -66,8 +81,8 @@ wrap_write (PyObject *self, PyObject *args)
   char *data;
   int len, rv;
 
-  if (!PyArg_ParseTuple(args, "s#", &data, &len)) 
-    return NULL;     
+  if (!PyArg_ParseTuple(args, "s#", &data, &len))
+    return NULL;
   Py_BEGIN_ALLOW_THREADS
   rv = pneum_write(data, len);
   Py_END_ALLOW_THREADS
@@ -117,16 +132,16 @@ initvpp_api (void)
 {
 #if PY_VERSION_HEX >= 0x03000000
   static struct PyModuleDef vpp_api_module = {
-# if PY_VERSION_HEX >= 0x03020000
+#if PY_VERSION_HEX >= 0x03020000
     PyModuleDef_HEAD_INIT,
-# else
+#else
     {
       PyObject_HEAD_INIT(NULL)
       NULL, /* m_init */
       0,    /* m_index */
       NULL, /* m_copy */
     },
-# endif
+#endif
     (char *) "vpp_api",
     NULL,
     -1,
diff --git a/vpp-api/python/vpp_papi/vpp_api_base.py b/vpp-api/python/vpp_papi/vpp_api_base.py
new file mode 100644
index 00000000000..a1ef87a3f0a
--- /dev/null
+++ b/vpp-api/python/vpp_papi/vpp_api_base.py
@@ -0,0 +1,97 @@
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Module storing all global variables, shared between main module and plugins
+#
+import threading
+
+#
+# Global variables
+#
+results = {}
+waiting_for_reply = False
+plugins = {}
+
+class ContextId(object):
+    def __init__(self):
+        self.context = 0
+    def __call__(self, id):
+        self.context += 1
+        return self.context
+get_context = ContextId()
+
+def waiting_for_reply_clear():
+    global waiting_for_reply
+    waiting_for_reply = False
+
+def waiting_for_reply_set():
+    global waiting_for_reply
+    waiting_for_reply = True
+
+def is_waiting_for_reply():
+    return waiting_for_reply
+
+def event_callback_set(callback):
+    global event_callback
+    event_callback = callback
+
+def event_callback_call(r):
+    global event_callback
+    event_callback(r)
+
+def results_event_set(context):
+    results[context]['e'].set()
+
+def results_event_clear(context):
+    results[context]['e'].clear()
+
+def results_event_wait(context, timeout):
+    results[context]['e'].wait(timeout)
+
+def results_set(context, r):
+    results[context]['r'] = r
+
+def results_append(context, r):
+    results[context]['r'].append(r)
+
+def is_results_context(context):
+    return context in results
+
+def is_results_more(context):
+    return 'm' in results[context]
+
+def results_more_set(context):
+    results[context]['m'] = True
+
+def results_prepare(context):
+    results[context] = {}
+    results[context]['e'] = threading.Event()
+    results[context]['e'].clear()
+    results[context]['r'] = []
+
+def results_get(context):
+    return results[context]['r']
+
+def plugin_register(name, func_table, name_to_id_table, version, msg_id_base_set):
+    plugins[name] = {}
+    p = plugins[name]
+    p['func_table'] = func_table
+    p['name_to_id_table'] = name_to_id_table
+    p['version'] = version
+    p['msg_id_base_set'] = msg_id_base_set
+
+def plugin_show():
+    for p in plugins:
+        print(p)
diff --git a/vpp-api/python/vpp_papi/vpp_papi.py b/vpp-api/python/vpp_papi/vpp_papi.py
new file mode 100644
index 00000000000..6a7a358f6cd
--- /dev/null
+++ b/vpp-api/python/vpp_papi/vpp_papi.py
@@ -0,0 +1,155 @@
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Import C API shared object
+#
+from __future__ import print_function
+
+import signal, logging, os, sys
+from struct import *
+
+scriptdir = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(scriptdir)
+import vpp_api
+from vpp_api_base import *
+
+# Import API definitions. The core VPE API is imported into main namespace
+import memclnt
+from vpe import *
+vpe = sys.modules['vpe']
+
+def msg_handler(msg):
+    if not msg:
+        logging.warning('vpp_api.read failed')
+        return
+
+    id = unpack('>H', msg[0:2])
+    logging.debug('Received message', id[0])
+    if id[0] == memclnt.VL_API_RX_THREAD_EXIT:
+        logging.info("We got told to leave")
+        return;
+
+    #
+    # Decode message and returns a tuple.
+    #
+    logging.debug('api_func', api_func_table[id[0]])
+    r = api_func_table[id[0]](msg)
+    if not r:
+        logging.warning('Message decode failed', id[0])
+        return
+
+    if 'context' in r._asdict():
+        if r.context > 0:
+            context = r.context
+
+    #
+    # XXX: Call provided callback for event
+    # Are we guaranteed to not get an event during processing of other messages?
+    # How to differentiate what's a callback message and what not? Context = 0?
+    #
+    if not is_waiting_for_reply():
+        event_callback_call(r)
+        return
+
+    #
+    # Collect results until control ping
+    #
+    if id[0] == vpe.VL_API_CONTROL_PING_REPLY:
+        results_event_set(context)
+        waiting_for_reply_clear()
+        return
+    if not is_results_context(context):
+        logging.warning('Not expecting results for this context', context)
+        return
+    if is_results_more(context):
+        results_append(context, r)
+        return
+
+    results_set(context, r)
+    results_event_set(context)
+    waiting_for_reply_clear()
+
+def connect(name):
+    signal.alarm(3) # 3 second
+    rv = vpp_api.connect(name, msg_handler)
+    signal.alarm(0)
+    logging.info("Connect:", rv)
+
+    #
+    # Assign message id space for plugins
+    #
+    plugin_map_plugins()
+
+    return rv
+
+def disconnect():
+    rv = vpp_api.disconnect()
+    logging.info("Disconnected")
+    return rv
+
+def register_event_callback(callback):
+    event_callback_set(callback)
+
+def plugin_name_to_id(plugin, name_to_id_table, base):
+    try:
+        m = globals()[plugin]
+    except KeyError:
+        m = sys.modules[plugin]
+    for name in name_to_id_table:
+        setattr(m, name, name_to_id_table[name] + base)
+
+def plugin_map_plugins():
+    for p in plugins:
+        if p == 'memclnt' or p == 'vpe':
+            continue
+
+        #
+        # Find base
+        # Update api table
+        #
+        version = plugins[p]['version']
+        name = p + '_' + format(version, '08x')
+        r = memclnt.get_first_msg_id(name.encode('ascii'))
+
+        ## TODO: Add error handling
+        if r.retval != 0:
+            print('Failed getting first msg id for:', p)
+            continue
+
+        # Set base
+        base = r.first_msg_id
+        msg_id_base_set = plugins[p]['msg_id_base_set']
+        msg_id_base_set(base)
+        plugins[p]['base'] = base
+        func_table = plugins[p]['func_table']
+        i = r.first_msg_id
+        for entry in func_table:
+            api_func_table.insert(i, entry)
+            i += 1
+        plugin_name_to_id(p, plugins[p]['name_to_id_table'], base)
+
+#
+# Set up core API
+#
+memclnt.msg_id_base_set(1)
+plugins['memclnt']['base'] = 1
+msg_id_base_set(len(plugins['memclnt']['func_table']) + 1)
+plugins['vpe']['base'] = len(plugins['memclnt']['func_table']) + 1
+api_func_table = []
+api_func_table.append(None)
+api_func_table[1:] = plugins['memclnt']['func_table'] + plugins['vpe']['func_table']
+plugin_name_to_id('memclnt', plugins['memclnt']['name_to_id_table'], 1)
+plugin_name_to_id('vpe', plugins['vpe']['name_to_id_table'], plugins['vpe']['base'])
+#logging.basicConfig(level=logging.DEBUG)
diff --git a/vpp/vpp-api/api.c b/vpp/vpp-api/api.c
index 1de1b55f722..2ba5ee4045f 100644
--- a/vpp/vpp-api/api.c
+++ b/vpp/vpp-api/api.c
@@ -2887,12 +2887,12 @@ vl_api_vnet_get_summary_stats_t_handler (vl_api_vnet_get_summary_stats_t * mp)
   }
   vnet_interface_counter_unlock (im);
 
-  /* Note: in HOST byte order! */
-  rmp->total_pkts[VLIB_RX] = total_pkts[VLIB_RX];
-  rmp->total_bytes[VLIB_RX] = total_bytes[VLIB_RX];
-  rmp->total_pkts[VLIB_TX] = total_pkts[VLIB_TX];
-  rmp->total_bytes[VLIB_TX] = total_bytes[VLIB_TX];
-  rmp->vector_rate = vlib_last_vector_length_per_node (sm->vlib_main);
+  rmp->total_pkts[VLIB_RX] = clib_host_to_net_u64 (total_pkts[VLIB_RX]);
+  rmp->total_bytes[VLIB_RX] = clib_host_to_net_u64 (total_bytes[VLIB_RX]);
+  rmp->total_pkts[VLIB_TX] = clib_host_to_net_u64 (total_pkts[VLIB_TX]);
+  rmp->total_bytes[VLIB_TX] = clib_host_to_net_u64 (total_bytes[VLIB_TX]);
+  rmp->vector_rate =
+    clib_host_to_net_u64 (vlib_last_vector_length_per_node (sm->vlib_main));
 
   vl_msg_api_send_shmem (q, (u8 *) & rmp);
 }
diff --git a/vppapigen/Makefile.am b/vppapigen/Makefile.am
index 42530015ea9..066e1c30d27 100644
--- a/vppapigen/Makefile.am
+++ b/vppapigen/Makefile.am
@@ -14,6 +14,7 @@
 AUTOMAKE_OPTIONS = foreign
 
 bin_PROGRAMS = vppapigen
+bin_SCRIPTS =  pyvppapigen.py
 
 BUILT_SOURCES = gram.h
 
diff --git a/vppapigen/lex.c b/vppapigen/lex.c
index 88744ff1029..b011044dd01 100644
--- a/vppapigen/lex.c
+++ b/vppapigen/lex.c
@@ -331,13 +331,17 @@ int main (int argc, char **argv)
         if (!strncmp (argv [curarg], "--python", 8)) {
             curarg++;
             if (curarg < argc) {
-                pythonfp = fopen (argv[curarg], "w");
+	        if (!strcmp(argv[curarg], "-")) {
+		    pythonfp = stdout;
+		} else {
+		    pythonfp = fopen(argv[curarg], "w");
+		    pythonfile = argv[curarg];
+		}
                 if (pythonfp == NULL) {
                     fprintf (stderr, "Couldn't open python output file %s\n",
                          argv[curarg]);
                     exit (1);
                 }
-                pythonfile = argv[curarg];
                 curarg++;
             } else {
                 fprintf(stderr, "Missing filename after --python\n");
diff --git a/vppapigen/node.c b/vppapigen/node.c
index e66fdce846a..420a128c00e 100644
--- a/vppapigen/node.c
+++ b/vppapigen/node.c
@@ -1339,6 +1339,11 @@ void generate_python (YYSTYPE a1, FILE *fp)
     np = np->peer;
   }
   fprintf (fp, "\n]\n");
+
+  /*
+   * API CRC signature
+   */
+  fprintf (fp, "vl_api_version = 0x%08x\n\n", (unsigned int)input_crc);
 }
 
 void generate(YYSTYPE a1)
diff --git a/vppapigen/pyvppapigen.py b/vppapigen/pyvppapigen.py
new file mode 100755
index 00000000000..e216169de3b
--- /dev/null
+++ b/vppapigen/pyvppapigen.py
@@ -0,0 +1,271 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import print_function
+import argparse, sys, os, importlib, pprint
+
+parser = argparse.ArgumentParser(description='VPP Python API generator')
+parser.add_argument('-i', '--input', action="store", dest="inputfile", type=argparse.FileType('r'))
+parser.add_argument('-c', '--cfile', action="store")
+args = parser.parse_args()
+
+#
+# Read API definitions file into vppapidefs
+#
+exec(args.inputfile.read())
+
+# https://docs.python.org/3/library/struct.html
+format_struct = {'u8': 'B',
+                 'u16' : 'H',
+                 'u32' : 'I',
+                 'i32' : 'i',
+                 'u64' : 'Q',
+                 'f64' : 'd',
+                 'vl_api_ip4_fib_counter_t' : 'IBQQ',
+                 'vl_api_ip6_fib_counter_t' : 'QQBQQ',
+                 };
+#
+# NB: If new types are introduced in vpe.api, these must be updated.
+#
+type_size = {'u8':   1,
+             'u16' : 2,
+             'u32' : 4,
+             'i32' : 4,
+             'u64' : 8,
+             'f64' : 8,
+             'vl_api_ip4_fib_counter_t' : 21,
+             'vl_api_ip6_fib_counter_t' : 33,
+};
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+def get_args(t):
+    argslist = []
+    for i in t:
+        if i[1][0] == '_':
+            argslist.append(i[1][1:])
+        else:
+            argslist.append(i[1])
+
+    return argslist
+
+def get_pack(f):
+    zeroarray = False
+    bytecount = 0
+    pack = ''
+    elements = 1
+    if len(f) is 3 or len(f) is 4:  # TODO: add support for variable length arrays (VPP-162)
+        size = type_size[f[0]]
+        bytecount += size * int(f[2])
+        # Check if we have a zero length array
+        if f[2] == '0':
+            # If len 3 zero array
+            elements = 0;
+            pack += format_struct[f[0]]
+            bytecount = size
+        elif size == 1:
+            n = f[2] * size
+            pack += str(n) + 's'
+        else:
+            pack += format_struct[f[0]] * int(f[2])
+            elements = int(f[2])
+    else:
+        bytecount += type_size[f[0]]
+        pack += format_struct[f[0]]
+    return (pack, elements, bytecount)
+
+
+'''
+def get_reply_func(f):
+    if f['name']+'_reply' in func_name:
+        return func_name[f['name']+'_reply']
+    if f['name'].find('_dump') > 0:
+        r = f['name'].replace('_dump','_details')
+        if r in func_name:
+            return func_name[r]
+    return None
+'''
+
+def footer_print():
+    print('''
+def msg_id_base_set(b):
+    global base
+    base = b
+
+import os
+name = os.path.splitext(os.path.basename(__file__))[0]
+    ''')
+    print(u"plugin_register(name, api_func_table, api_name_to_id,", vl_api_version, ", msg_id_base_set)")
+
+def api_table_print(name, i):
+    msg_id_in = 'VL_API_' + name.upper()
+    fstr = name + '_decode'
+    print('api_func_table.append(' + fstr + ')')
+    print('api_name_to_id["' + msg_id_in + '"] =', i)
+    print('')
+
+def encode_print(name, id, t):
+    total = 0
+    args = get_args(t)
+    pack = '>'
+    for i, f in enumerate(t):
+        p, elements, size = get_pack(f)
+        pack += p
+        total += size
+
+    if name.find('_dump') > 0:
+        multipart = True
+    else:
+        multipart = False
+
+    if len(args) < 4:
+        print(u"def", name + "(async = False):")
+    else:
+        print(u"def", name + "(" + ', '.join(args[3:]) + ", async = False):")
+    print(u"    global base")
+    print(u"    context = get_context(base + " + id + ")")
+
+    print('''
+    results_prepare(context)
+    waiting_for_reply_set()
+    ''')
+    if multipart == True:
+        print(u"    results_more_set(context)")
+
+    ### TODO deal with zeroarray!!!
+    #if zeroarray == True:
+    #    print(u"    vpp_api.write(pack('" + pack + "', " + id + ", 0, context, " + ', '.join(args[3:-1]) + ") + " + args[-1] + ")")
+    #else:
+    print(u"    vpp_api.write(pack('" + pack + "', base + " + id + ", 0, context, " + ', '.join(args[3:]) + "))")
+
+    if multipart == True:
+        print(u"    vpp_api.write(pack('>HII', VL_API_CONTROL_PING, 0, context))")
+
+    print('''
+    if not async:
+        results_event_wait(context, 5)
+        return results_get(context)
+    return context
+    ''')
+
+def get_normal_pack(t, i, pack, offset):
+    while t:
+        f = t.pop(0)
+        i += 1
+        if len(f) >= 3:
+            return t, i, pack, offset, f
+        p, elements, size = get_pack(f)
+        pack += p
+        offset += size
+    return t, i, pack, offset, None
+
+def decode_print(name, t):
+    #
+    # Generate code for each element
+    #
+    print(u'def ' + name + u'_decode(msg):')
+    total = 0
+    args = get_args(t)
+    print(u"    n = namedtuple('" + name + "', '" + ', '.join(args) + "')")
+    print(u"    res = []")
+
+    pack = '>'
+    start = 0
+    end = 0
+    offset = 0
+    t = list(t)
+    i = 0
+    while t:
+        t, i, pack, offset, array = get_normal_pack(t, i, pack, offset)
+        if array:
+            p, elements, size = get_pack(array)
+
+            # Byte string
+            if elements > 0 and type_size[array[0]] == 1:
+                pack += p
+                offset += size * elements
+                continue
+
+            # Dump current pack string
+            if pack != '>':
+                print(u"    tr = unpack_from('" + pack + "', msg[" + str(start) + ":])")
+                print(u"    res.extend(list(tr))")
+                start += offset
+            pack = '>'
+
+            if elements == 0:
+                # This has to be the last element
+                if len(array) == 3:
+                    print(u"    res.append(msg[" + str(offset) + ":])")
+                    if len(t) > 0:
+                        eprint('WARNING: Variable length array must be last element in message', name, array)
+
+                    continue
+                if size == 1 or len(p) == 1:
+                    # Do it as a bytestring.
+                    if p == 'B':
+                        p = 's'
+                    # XXX: Assume that length parameter is the previous field. Add validation.
+                    print(u"    c = res[" + str(i - 2) + "]")
+                    print(u"    tr = unpack_from('>' + str(c) + '" + p + "', msg[" + str(start) + ":])")
+                    print(u"    res.append(tr)")
+                    continue
+                print(u"    tr2 = []")
+                print(u"    offset = " + str(total))
+                print(u"    for j in range(res[" + str(i - 2) + "]):")
+                print(u"        tr2.append(unpack_from('>" + p + "', msg[" + str(start) + ":], offset))")
+                print(u"        offset += " + str(size))
+                print(u"    res.append(tr2)")
+                continue
+
+            # Missing something!!
+            print(u"    tr = unpack_from('>" + p + "', msg[" + str(start) + ":])")
+            start += size
+
+            print(u"    res.append(tr)")
+
+    if pack != '>':
+        print(u"    tr = unpack_from('" + pack + "', msg[" + str(start) + ":])")
+        print(u"    res.extend(list(tr))")
+    print(u"    return n._make(res)")
+    print('')
+
+#
+# Generate the main Python file
+#
+def main():
+    print('''
+#
+# AUTO-GENERATED FILE. PLEASE DO NOT EDIT.
+#
+from vpp_api_base import *
+from struct import *
+from collections import namedtuple
+import vpp_api
+api_func_table = []
+api_name_to_id = {}
+    ''')
+
+    for i, a in enumerate(vppapidef):
+        name = a[0]
+        encode_print(name, str(i), a[1:])
+        decode_print(name, a[1:])
+        api_table_print(name, i)
+    footer_print()
+
+if __name__ == "__main__":
+    main()