Luajit API and some examples

Change-Id: Ia140c4750f06870c40b7058c4afb2e20ca633a49
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>

1 files changed, 752 insertions, 0 deletions

diff --git a/vpp-api/lua/vpp-lapi.lua b/vpp-api/lua/vpp-lapi.lua
new file mode 100644
index 00000000000..4f2e3f53d59
--- /dev/null
+++ b/vpp-api/lua/vpp-lapi.lua
@@ -0,0 +1,752 @@
+--[[
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+local vpp = {}
+
+local ffi = require("ffi")
+
+--[[
+
+The basic type definitions. A bit of weird gymnastic with
+unionization of the hton* and ntoh* functions results
+is to make handling of signed and unsigned types a bit cleaner,
+essentially building typecasting into a C union.
+
+The vl_api_opaque_message_t is a synthetic type assumed to have
+enough storage to hold the entire API message regardless of the type.
+During the operation it is casted to the specific message struct types.
+
+]]
+
+
+ffi.cdef([[
+
+typedef uint8_t u8;
+typedef int8_t i8;
+typedef uint16_t u16;
+typedef int16_t i16;
+typedef uint32_t u32;
+typedef int32_t i32;
+typedef uint64_t u64;
+typedef int64_t i64;
+typedef double f64;
+typedef float f32;
+
+#pragma pack(1)
+typedef union {
+  u16 u16;
+  i16 i16;
+} lua_ui16t;
+
+#pragma pack(1)
+typedef union {
+  u32 u32;
+  i32 i32;
+} lua_ui32t;
+
+u16 ntohs(uint16_t hostshort);
+u16 htons(uint16_t hostshort);
+u32 htonl(uint32_t along);
+u32 ntohl(uint32_t along);
+void *memset(void *s, int c, size_t n);
+void *memcpy(void *dest, void *src, size_t n);
+
+#pragma pack(1)
+typedef struct _vl_api_opaque_message {
+  u16 _vl_msg_id;
+  u8  data[65536];
+} vl_api_opaque_message_t;
+]])
+
+
+-- CRC-based version stuff
+
+local crc32c_table = ffi.new('const uint32_t[256]',
+  { 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4,
+  0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
+  0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
+  0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24,
+  0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B,
+  0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384,
+  0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54,
+  0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B,
+  0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A,
+  0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35,
+  0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5,
+  0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA,
+  0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45,
+  0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A,
+  0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A,
+  0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595,
+  0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48,
+  0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957,
+  0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687,
+  0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198,
+  0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927,
+  0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38,
+  0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8,
+  0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7,
+  0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096,
+  0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789,
+  0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859,
+  0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46,
+  0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9,
+  0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6,
+  0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36,
+  0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829,
+  0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C,
+  0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93,
+  0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043,
+  0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C,
+  0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3,
+  0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC,
+  0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C,
+  0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033,
+  0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652,
+  0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D,
+  0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D,
+  0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982,
+  0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D,
+  0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622,
+  0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2,
+  0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED,
+  0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530,
+  0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F,
+  0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF,
+  0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0,
+  0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F,
+  0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540,
+  0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90,
+  0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F,
+  0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE,
+  0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1,
+  0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321,
+  0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E,
+  0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81,
+  0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
+  0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
+  0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351 }
+);
+
+local function CRC8(crc, d)
+  return bit.bxor(bit.rshift(crc, 8), crc32c_table[bit.band(0xff, bit.bxor(crc, d))])
+end
+
+local function CRC16(crc, d)
+  crc = CRC8(crc, bit.band(d, 0xFF))
+  d = bit.rshift(d, 8)
+  crc = CRC8(crc, bit.band(d, 0xFF))
+  return crc
+end
+
+local function string_crc(str, crc)
+  for i=1,#str do
+    -- print("S", i, string.byte(str, i), string.char(string.byte(str, i)))
+    crc = CRC8(crc, string.byte(str, i))
+  end
+  return crc
+end
+
+local tokens = {
+  { ["match"] =' ', ["act"]             = { }  },
+  { ["match"] ='\n', ["act"]             = { }  },
+  { ["match"] ="manual_endian", ["act"]  = { "NODE_MANUAL_ENDIAN", "MANUAL_ENDIAN",    276 } },
+  { ["match"] ="define", ["act"]         = { "NODE_DEFINE",        "DEFINE",           267 } },
+  { ["match"] ="dont_trace", ["act"]     = { "NODE_DONT_TRACE",    "DONT_TRACE",       279 } },
+  { ["match"] ="f64", ["act"]            = { "NODE_F64",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="i16", ["act"]            = { "NODE_I16",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="i32", ["act"]            = { "NODE_I32",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="i64", ["act"]            = { "NODE_I64",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="i8", ["act"]             = { "NODE_I8",            "PRIMTYPE",         string_crc } },
+  { ["match"] ="manual_print", ["act"]   = { "NODE_MANUAL_PRINT",  "MANUAL_PRINT",     275 } },
+  { ["match"] ="noversion", ["act"]      = { "NODE_NOVERSION",     "NOVERSION",        274 } },
+  { ["match"] ="packed", ["act"]         = { "NODE_PACKED",        "TPACKED",          266 } },
+  { ["match"] ="typeonly", ["act"]       = { "NODE_TYPEONLY",      "TYPEONLY",         278 } },
+  { ["match"] ="u16", ["act"]            = { "NODE_U16",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="u32", ["act"]            = { "NODE_U32",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="u64", ["act"]            = { "NODE_U64",           "PRIMTYPE",         string_crc } },
+  { ["match"] ="u8", ["act"]             = { "NODE_U8",            "PRIMTYPE",         string_crc } },
+  { ["match"] ="union", ["act"]          = { "NODE_UNION",         "UNION",            271 } },
+  { ["match"] ="uword", ["act"]          = { "NODE_UWORD",         "PRIMTYPE",         string_crc } },
+  { ["match"] ="%(", ["act"]             = { "NODE_LPAR",          "LPAR",             259 } },
+  { ["match"] ="%)", ["act"]             = { "NODE_RPAR",          "RPAR",             258 } },
+  { ["match"] =";", ["act"]              = { "NODE_SEMI",          "SEMI",             260 } },
+  { ["match"] ="%[", ["act"]             = { "NODE_LBRACK",        "LBRACK",           261 } },
+  { ["match"] ="%]", ["act"]             = { "NODE_RBRACK",        "RBRACK",           262 } },
+  { ["match"] ="%{", ["act"]             = { "NODE_LCURLY",        "LCURLY",           268 } },
+  { ["match"] ="%}", ["act"]             = { "NODE_RCURLY",        "RCURLY",           269 } },
+  { ["match"] ='%b""', ["act"]           = { "NODE_STRING",        "STRING",           string_crc } },
+  { ["match"] ='%b@@', ["act"]           = { "NODE_HELPER",        "HELPER_STRING",    string_crc } },
+  -- TODO: \ must be consumed
+  { ["match"] ='[_a-zA-Z][_a-zA-Z0-9]*',
+                       ["act"]           = { "NODE_NAME",          "NAME",             string_crc } },
+  { ["match"] ='[0-9]+', ["act"]         = { "NODE_NUMBER",        "NUMBER",           string_crc } },
+  { ["match"] ='#[^\n]+', ["act"]            = { "NODE_PRAGMA",        "PRAGMA",           nil } },
+}
+
+
+function vpp.crc_version_string(data)
+  local input_crc = 0
+  -- Get rid of comments
+  data = data:gsub("/%*.-%*/", "")
+  data = data:gsub("//[^\n]+", "")
+  -- print(data)
+  idx = 1
+  while (true) do
+    local matched = nil
+    for k, v in ipairs(tokens) do
+      if not matched then
+        local x, y, cap = string.find(data, v["match"], idx)
+        if x == idx then
+          matched = { ["node"] = v["act"], ["x"] = x, ["y"] = y, ["cap"] = cap, ["chars"] = string.sub(data, x, y)  }
+          -- print(k, v, x, y, cap, matched.chars, matched.node[0] )
+        end
+      end
+    end
+    if matched then
+      idx = idx + (matched.y - matched.x + 1)
+      if matched.node[1] then
+        local act = matched.node[3]
+        if type(act) == "function" then
+          input_crc = act(matched.chars, input_crc)
+        elseif type(act) == "number" then
+          input_crc = CRC16(input_crc, act)
+        end
+        -- print(vpp.dump(matched))
+      end
+    else
+      -- print("NOT MATCHED!")
+      local crc = CRC16(input_crc, 0xFFFFFFFF)
+      return string.sub(string.format("%x", crc), -8)
+    end
+  end
+end
+
+
+function vpp.dump(o)
+   if type(o) == 'table' then
+      local s = '{ '
+      for k,v in pairs(o) do
+         if type(k) ~= 'number' then k = '"'..k..'"' end
+         s = s .. '['..k..'] = ' .. vpp.dump(v) .. ','
+      end
+      return s .. '} '
+   else
+      return tostring(o)
+   end
+end
+
+function vpp.hex_dump(buf)
+  local ret = {}
+  for i=1,math.ceil(#buf/16) * 16 do
+    if (i-1) % 16 == 0 then table.insert(ret, string.format('%08X  ', i-1)) end
+    table.insert(ret, ( i > #buf and '   ' or string.format('%02X ', buf:byte(i)) ))
+    if i %  8 == 0 then table.insert(ret, ' ') end
+    if i % 16 == 0 then table.insert(ret, buf:sub(i-16+1, i):gsub('%c','.')..'\n' ) end
+  end
+  return table.concat(ret)
+end
+
+
+function vpp.c_str(text_in)
+  local text = text_in -- \000 will be helpfully added by ffi.copy
+  local c_str = ffi.new("char[?]", #text+1)
+  ffi.copy(c_str, text)
+  return c_str
+end
+
+
+function vpp.init(vpp, args)
+  local pneum_api = args.pneum_api or [[
+ int cough_pneum_attach(char *pneum_path, char *cough_path);
+ int pneum_connect(char *name, char *chroot_prefix);
+ int pneum_connect_sync(char *name, char *chroot_prefix);
+ int pneum_disconnect(void);
+ int pneum_read(char **data, int *l);
+ int pneum_write(char *data, int len);
+
+void pneum_data_free(char *data);
+]]
+
+  vpp.pneum_path = args.pneum_path
+  ffi.cdef(pneum_api)
+  local init_res = 0
+  if pcall(function()
+		     vpp.cough_path = args.cough_path or "./libcough.so"
+                     vpp.cough = ffi.load(vpp.cough_path)
+           end) then
+    pcall(function()
+      if(vpp.cough.cough_pneum_attach) then
+        vpp.pneum_is_cough = true
+        print("libcough detected\n")
+        init_res = vpp.cough.cough_pneum_attach(vpp.c_str(vpp.pneum_path), vpp.c_str(vpp.cough_path))
+        vpp.pneum = vpp.cough
+      end
+    end)
+  else
+    vpp.pneum = ffi.load(vpp.pneum_path)
+  end
+  if (init_res < 0) then
+    return nil
+  end
+
+  vpp.next_msg_num = 1
+  vpp.msg_name_to_number = {}
+  vpp.msg_name_to_fields = {}
+  vpp.msg_number_to_name = {}
+  vpp.msg_number_to_type = {}
+  vpp.msg_number_to_pointer_type = {}
+  vpp.c_type_to_fields = {}
+  vpp.events = {}
+  vpp.plugin_version = {}
+  vpp.is_connected = false
+
+
+  vpp.t_lua2c = {}
+  vpp.t_c2lua = {}
+  vpp.t_lua2c["u8"] = function(c_type, src, dst_c_ptr)
+    if type(src) == "string" then
+      -- ffi.copy adds a zero byte at the end. Grrr.
+      -- ffi.copy(dst_c_ptr, src)
+      ffi.C.memcpy(dst_c_ptr, vpp.c_str(src), #src)
+      return(#src)
+    elseif type(src) == "table" then
+      for i,v in ipairs(src) do
+        ffi.cast("u8 *", dst_c_ptr)[i-1] = v
+      end
+      return(#src)
+    else
+      return 1, src -- ffi.cast("u8", src)
+    end
+  end
+  vpp.t_c2lua["u8"] = function(c_type, src_ptr, src_len)
+    if src_len then
+      return ffi.string(src_ptr, src_len)
+    else
+      return (tonumber(src_ptr))
+    end
+  end
+
+  vpp.t_lua2c["u16"] = function(c_type, src, dst_c_ptr)
+    if type(src) == "table" then
+      for i,v in ipairs(src) do
+        ffi.cast("u16 *", dst_c_ptr)[i-1] = ffi.C.htons(v)
+      end
+      return(2 * #src)
+    else
+      return 2, (ffi.C.htons(src))
+    end
+  end
+  vpp.t_c2lua["u16"] = function(c_type, src_ptr, src_len)
+    if src_len then
+      local out = {}
+      for i = 0,src_len-1 do
+        out[i+1] = tonumber(ffi.C.ntohs(src_ptr[i]))
+      end
+      return out
+    else
+      return (tonumber(ffi.C.ntohs(src_ptr)))
+    end
+  end
+
+  vpp.t_lua2c["u32"] = function(c_type, src, dst_c_ptr)
+    if type(src) == "table" then
+      for i,v in ipairs(src) do
+        ffi.cast("u32 *", dst_c_ptr)[i-1] = ffi.C.htonl(v)
+      end
+      return(4 * #src)
+    else
+      return 4, (ffi.C.htonl(src))
+    end
+  end
+  vpp.t_c2lua["u32"] = function(c_type, src_ptr, src_len)
+    if src_len then
+      local out = {}
+      for i = 0,src_len-1 do
+        out[i+1] = tonumber(ffi.C.ntohl(src_ptr[i]))
+      end
+      return out
+    else
+      return (tonumber(ffi.C.ntohl(src_ptr)))
+    end
+  end
+  vpp.t_lua2c["i32"] = function(c_type, src, dst_c_ptr)
+    if type(src) == "table" then
+      for i,v in ipairs(src) do
+        ffi.cast("i32 *", dst_c_ptr)[i-1] = ffi.C.htonl(v)
+      end
+      return(4 * #src)
+    else
+      return 4, (ffi.C.htonl(src))
+    end
+  end
+  vpp.t_c2lua["i32"] = function(c_type, src_ptr, src_len)
+    local ntohl = function(src)
+      local u32val = ffi.cast("u32", src)
+      local ntohlval = (ffi.C.ntohl(u32val))
+      local out = tonumber(ffi.cast("i32", ntohlval + 0LL))
+      return out
+    end
+    if src_len then
+      local out = {}
+      for i = 0,src_len-1 do
+        out[i+1] = tonumber(ntohl(src_ptr[i]))
+      end
+    else
+      return (tonumber(ntohl(src_ptr)))
+    end
+  end
+
+  vpp.t_lua2c["u64"] = function(c_type, src, dst_c_ptr)
+    if type(src) == "table" then
+      for i,v in ipairs(src) do
+        ffi.cast("u64 *", dst_c_ptr)[i-1] = v --- FIXME ENDIAN
+      end
+      return(8 * #src)
+    else
+      return 8, ffi.cast("u64", src) --- FIXME ENDIAN
+    end
+  end
+  vpp.t_c2lua["u64"] = function(c_type, src_ptr, src_len)
+    if src_len then
+      local out = {}
+      for i = 0,src_len-1 do
+        out[i+1] = tonumber(src_ptr[i]) -- FIXME ENDIAN
+      end
+      return out
+    else
+      return (tonumber(src_ptr)) --FIXME ENDIAN
+    end
+  end
+
+
+
+
+  vpp.t_lua2c["__MSG__"] = function(c_type, src, dst_c_ptr)
+    local dst = ffi.cast(c_type .. " *", dst_c_ptr)
+    local additional_len = 0
+    local fields_info = vpp.c_type_to_fields[c_type]
+    -- print("__MSG__ type: " .. tostring(c_type))
+    ffi.C.memset(dst_c_ptr, 0, ffi.sizeof(dst[0]))
+    -- print(vpp.dump(fields_info))
+    -- print(vpp.dump(src))
+    for k,v in pairs(src) do
+      local field = fields_info[k]
+      if not field then
+        print("ERROR: field " .. tostring(k) .. " in message " .. tostring(c_type) .. " is unknown")
+      end
+      local lua2c = vpp.t_lua2c[field.c_type]
+      -- print("__MSG__ field " .. tostring(k) .. " : " .. vpp.dump(field))
+      -- if the field is not an array type, try to coerce the argument to a number
+      if not field.array and type(v) == "string" then
+        v = tonumber(v)
+      end
+      if not lua2c then
+        print("__MSG__ " .. tostring(c_type) .. " t_lua2c: can not store field " .. field.name ..
+              " type " .. field.c_type .. " dst " .. tostring(dst[k]))
+        return 0
+      end
+      local len = 0
+      local val = nil
+      if field.array and (type(v) == "table") then
+        print("NTFY: field " .. tostring(k) .. " in message " .. tostring(c_type) .. " is an array")
+        for field_i, field_v in ipairs(v) do
+          print("NTFY: setting member#" .. tostring(field_i) .. " to value " .. vpp.dump(field_v))
+          local field_len, field_val = lua2c(field.c_type, field_v, dst[k][field_i-1])
+          len = len + field_len
+        end
+      else
+        len, val = lua2c(field.c_type, v, dst[k])
+      end
+      if not field.array then
+        dst[k] = val
+      else
+        if 0 == field.array then
+          additional_len = additional_len + len
+          -- print("Adding " .. tostring(len) .. " bytes due to field " .. tostring(field.name))
+          -- If there is a variable storing the length
+          -- and the input table does not set it, do magic
+          if field.array_size and not src[field.array_size] then
+            local size_field = fields_info[field.array_size]
+            if size_field then
+              dst[field.array_size] = vpp.t_c2lua[size_field.c_type](size_field.c_type, len)
+            end
+          end
+        end
+      end
+      -- print("Full message:\n" .. vpp.hex_dump(ffi.string(ffi.cast('void *', req_store_cache), 64)))
+    end
+    return (ffi.sizeof(dst[0])+additional_len)
+  end
+
+  vpp.t_c2lua["__MSG__"] = function(c_type, src_ptr, src_len)
+    local out = {}
+    local reply_typed_ptr = ffi.cast(c_type .. " *", src_ptr)
+    local field_desc = vpp.c_type_to_fields[c_type]
+    if src_len then
+      for i = 0,src_len-1 do
+        out[i+1] = vpp.t_c2lua[c_type](c_type, src_ptr[i])
+      end
+      return out
+    end
+
+    for k, v in pairs(field_desc) do
+      local v_c2lua = vpp.t_c2lua[v.c_type]
+      if v_c2lua then
+        local len = v.array
+        -- print(dump(v))
+        if len then
+          local len_field_name = k .. "_length"
+          local len_field = field_desc[len_field_name]
+          if (len_field) then
+            local real_len = vpp.t_c2lua[len_field.c_type](len_field.c_type, reply_typed_ptr[len_field_name])
+            out[k] =  v_c2lua(v.c_type, reply_typed_ptr[k], real_len)
+          elseif len == 0 then
+            -- check if len = 0, then must be a field which contains the size
+            len_field =  field_desc[v.array_size]
+            local real_len = vpp.t_c2lua[len_field.c_type](len_field.c_type, reply_typed_ptr[v.array_size])
+            -- print("REAL length: " .. vpp.dump(v) .. " : " .. tostring(real_len))
+            out[k] = v_c2lua(v.c_type, reply_typed_ptr[k], real_len)
+          else
+            -- alas, just stuff the entire array
+            out[k] = v_c2lua(v.c_type, reply_typed_ptr[k], len)
+          end
+        else
+          out[k] =  v_c2lua(v.c_type, reply_typed_ptr[k])
+        end
+      else
+        out[k] = "<no accessor function for type " .. tostring(v.c_type) .. ">"
+      end
+      -- print(k, out[k])
+    end
+    return out
+  end
+
+  return vpp
+end
+
+function vpp.connect(vpp, client_name)
+    local name = "lua_client"
+    if client_name then
+      name = client_name
+    end
+    local ret = vpp.pneum.pneum_connect_sync(vpp.c_str(client_name), nil)
+    if tonumber(ret) == 0 then
+      vpp.is_connected = true
+    end
+  end
+
+function vpp.disconnect(vpp)
+    vpp.pneum.pneum_disconnect()
+  end
+
+function vpp.consume_api(vpp, path, plugin_name)
+    -- print("Consuming the VPP api from "..path)
+    local ffii = {}
+    local f = io.open(path, "r")
+    if not f then
+      print("Could not open " .. path)
+      return nil
+    end
+    local data = f:read("*all")
+    -- Remove all C comments
+    data = data:gsub("/%*.-%*/", "")
+    if vpp.is_connected and not plugin_name then
+      print(path .. ": must specify plugin name!")
+      return
+    end
+    if plugin_name then
+      vpp.plugin_version[plugin_name] = vpp.crc_version_string(data)
+      local full_plugin_name = plugin_name .. "_" .. vpp.plugin_version[plugin_name]
+      local reply = vpp:api_call("get_first_msg_id", { name = full_plugin_name } )
+      vpp.next_msg_num = tonumber(reply[1].first_msg_id)
+      print("Plugin " .. full_plugin_name .. " first message is " .. tostring(vpp.next_msg_num))
+    end
+    -- print ("data len: ", #data)
+    data = data:gsub("\n(.-)(%S+)%s*{([^}]*)}", function (preamble, name, members)
+      local _, typeonly = preamble:gsub("typeonly", "")
+      local maybe_msg_id_field = { [0] = "u16 _vl_msg_id;", "" }
+      local onedef = "\n\n#pragma pack(1)\ntypedef struct _vl_api_"..name.. " {\n" ..
+	   -- "   u16 _vl_msg_id;" ..
+           maybe_msg_id_field[typeonly] ..
+	   members:gsub("%[[a-zA-Z_]+]", "[0]") ..
+	   "} vl_api_" .. name .. "_t;"
+
+      local c_type = "vl_api_" .. name .. "_t"
+
+      local fields = {}
+      -- vpp.msg_name_to_fields[name] = fields
+      -- print("CTYPE " .. c_type)
+      vpp.c_type_to_fields[c_type] = fields
+      vpp.t_lua2c[c_type] = vpp.t_lua2c["__MSG__"]
+      vpp.t_c2lua[c_type] = vpp.t_c2lua["__MSG__"]
+      local mirec = { name = "_vl_msg_id", c_type = "u16", array = nil, array_size = nil }
+      if typeonly == 0 then
+        fields[mirec.name] = mirec
+      end
+
+      -- populate the field reflection table for the message
+      -- sets the various type information as well as the accessors for lua<->C conversion
+      members:gsub("(%S+)%s+(%S+);", function (fieldtype, fieldname)
+          local fieldcount = nil
+          local fieldcountvar = nil
+          -- data = data:gsub("%[[a-zA-Z_]+]", "[0]")
+          fieldname = fieldname:gsub("(%b[])", function(cnt)
+              fieldcount = tonumber(cnt:sub(2, -2));
+              if not fieldcount then
+                fieldcount = 0
+                fieldcountvar = cnt:sub(2, -2)
+              end
+              return ""
+            end)
+	  local fieldrec = { name = fieldname, c_type = fieldtype, array = fieldcount, array_size = fieldcountvar }
+          if fieldcount then
+            if fieldtype == "u8" then
+              -- any array of bytes is treated as a string
+            elseif vpp.t_lua2c[fieldtype] then
+              -- print("Array of " .. fieldtype .. " is ok!")
+            else
+              print("Unknown array type: ", name,  " : " , fieldname, " : ", fieldtype, ":", fieldcount, ":", fieldcountvar)
+            end
+          end
+	  fields[fieldname] = fieldrec
+	end)
+
+      -- print(dump(fields))
+
+      if typeonly == 0 then
+	local this_message_number = vpp.next_msg_num
+	vpp.next_msg_num = vpp.next_msg_num + 1
+	vpp.msg_name_to_number[name] = this_message_number
+	vpp.msg_number_to_name[this_message_number] = name
+	vpp.msg_number_to_type[this_message_number] = "vl_api_" .. name .. "_t"
+	vpp.msg_number_to_pointer_type[this_message_number] = vpp.msg_number_to_type[this_message_number] .. " *"
+	onedef = onedef .. "\n\n enum { vl_msg_" .. name .. " = " .. this_message_number .. " };\n\n"
+      end
+      table.insert(ffii, onedef);
+      return "";
+      end)
+    local cdef = table.concat(ffii)
+    -- print(cdef)
+    ffi.cdef(cdef)
+  end
+
+
+function vpp.lua2c(vpp, c_type, src, dst_c_ptr)
+  -- returns the number of bytes written to memory pointed by dst
+  local lua2c = vpp.t_lua2c[c_type]
+  if lua2c then
+    return(lua2c(c_type, src, dst_c_ptr))
+  else
+    print("vpp.lua2c: do not know how to store type " .. c_type)
+    return 0
+  end
+end
+
+function vpp.c2lua(vpp, c_type, src_ptr, src_len)
+  -- returns the lua data structure
+  local c2lua = vpp.t_c2lua[c_type]
+  if c2lua then
+    return(c2lua(c_type, src_ptr, src_len))
+  else
+    print("vpp.c2lua: do not know how to load type " .. c_type)
+    return nil
+  end
+end
+
+local req_store_cache = ffi.new("vl_api_opaque_message_t[1]")
+
+function vpp.api_write(vpp, api_name, req_table)
+    local msg_num = vpp.msg_name_to_number[api_name]
+    if not msg_num then
+      print ("API call "..api_name.." is not known")
+      return nil
+    end
+
+    if not req_table then
+      req_table = {}
+    end
+    req_table._vl_msg_id = msg_num
+
+    local packed_len = vpp:lua2c(vpp.msg_number_to_type[msg_num], req_table, req_store_cache)
+    if vpp.debug_dump then
+      print("Write Message length: " .. tostring(packed_len) .. "\n" .. vpp.hex_dump(ffi.string(ffi.cast('void *', req_store_cache), packed_len)))
+    end
+
+    res = vpp.pneum.pneum_write(ffi.cast('void *', req_store_cache), packed_len)
+    return res
+  end
+
+local rep_store_cache = ffi.new("vl_api_opaque_message_t *[1]")
+local rep_len_cache = ffi.new("int[1]")
+
+function vpp.api_read(vpp)
+    local rep_type = "vl_api_opaque_message_t"
+    local rep = rep_store_cache
+    local replen = rep_len_cache
+    res = vpp.pneum.pneum_read(ffi.cast("void *", rep), replen)
+    if vpp.debug_dump then
+      print("Read Message length: " .. tostring(replen[0]) .. "\n" .. vpp.hex_dump(ffi.string(ffi.cast('void *', rep[0]), replen[0])))
+    end
+
+    local reply_msg_num = ffi.C.ntohs(rep[0]._vl_msg_id)
+    local reply_msg_name = vpp.msg_number_to_name[reply_msg_num]
+
+    local reply_typed_ptr = ffi.cast(vpp.msg_number_to_pointer_type[reply_msg_num], rep[0])
+    local out = vpp:c2lua(vpp.msg_number_to_type[reply_msg_num], rep[0], nil, replen[0])
+    if type(out) == "table" then
+      out["luaapi_message_name"] = reply_msg_name
+    end
+
+    vpp.pneum.pneum_data_free(ffi.cast('void *',rep[0]))
+
+    return reply_msg_name, out
+  end
+
+function vpp.api_call(vpp, api_name, req_table, options_in)
+    local msg_num = vpp.msg_name_to_number[api_name]
+    local end_message_name = api_name .."_reply"
+    local replies = {}
+    local cstruct = ""
+    local options = options_in or {}
+    if msg_num then
+      vpp:api_write(api_name, req_table)
+      if not vpp.msg_name_to_number[end_message_name] or options.force_ping then
+        end_message_name = "control_ping_reply"
+        vpp:api_write("control_ping")
+      end
+      repeat
+        reply_message_name, reply = vpp:api_read()
+        if reply and not reply.context then
+          -- there may be async events inbetween
+          table.insert(vpp.events, reply)
+        else
+          if reply_message_name ~= "control_ping_reply" then
+            -- do not insert the control ping encapsulation
+            table.insert(replies, reply)
+          end
+        end
+        -- print(reply)
+      until reply_message_name == end_message_name
+    else
+      print(api_name .. " is an unknown API call")
+      return nil
+    end
+    return replies
+  end
+
+return vpp