Python API: Add back support for old-style VLA, e.g. mask[0]

Change-Id: I5be6bfe522e5ea93934f0ddf75d4e4707376dc0c
Signed-off-by: Ole Troan <ot@cisco.com>

3 files changed, 144 insertions, 51 deletions

diff --git a/vpp-api/python/tests/test_vpp_papi2.py b/vpp-api/python/tests/test_vpp_papi2.py
index f5e219adb65..18f6b6866a8 100755
--- a/vpp-api/python/tests/test_vpp_papi2.py
+++ b/vpp-api/python/tests/test_vpp_papi2.py
@@ -46,7 +46,7 @@ class TestPAPI(unittest.TestCase):
 
     def test_adding_new_message_object(self):
         p = json.loads(TestPAPI.show_version_msg)
-        msglist = VPP([])
+        msglist = VPP(testmode=json)
         msgdef = msglist.add_message(p[0], p[1:])
 
         # Verify that message can be retrieved
@@ -61,13 +61,13 @@ class TestPAPI(unittest.TestCase):
 
     def test_adding_new_message_object_with_array(self):
         p = json.loads(TestPAPI.ip_address_details_msg)
-        msglist = VPP([])
+        msglist = VPP(testmode=True)
         msglist.add_message(p[0], p[1:])
 
         self.assertTrue(msglist['ip_address_details'])
 
     def test_message_to_bytes(self):
-        msglist = VPP([])
+        msglist = VPP(testmode=True)
         p = json.loads(TestPAPI.show_version_msg)
         msgdef = msglist.add_message(p[0], p[1:])
 
@@ -133,12 +133,11 @@ class TestPAPI(unittest.TestCase):
         with_type_msg = '''["with_type_msg",
             ["u32", "length"],
             ["u16", "list", 0, "length"],
-            ["ip4_fib_counter", "counter"]
-
+            ["vl_api_ip4_fib_counter_t", "counter"]
         ]'''
 
         # Add new type
-        msglist = VPP([])
+        msglist = VPP(testmode=True)
         p = json.loads(counter_type)
         msglist.add_type(p[0], p[1:])
         p = json.loads(with_type_msg)
@@ -163,12 +162,12 @@ class TestPAPI(unittest.TestCase):
         with_type_msg = '''["with_type_msg",
             ["u32", "length"],
             ["u16", "list", 0, "length"],
-            ["ip4_fib_counter", "counter", 2]
+            ["vl_api_ip4_fib_counter_t", "counter", 2]
 
         ]'''
 
         # Add new type
-        msglist = VPP([])
+        msglist = VPP(testmode=True)
         p = json.loads(counter_type)
         msglist.add_type(p[0], p[1:])
         p = json.loads(with_type_msg)
@@ -185,7 +184,7 @@ class TestPAPI(unittest.TestCase):
 
         with_type_variable_msg = '''["with_type_variable_msg",
             ["u32", "length"],
-            ["ip4_fib_counter", "counter", 0, "length"]
+            ["vl_api_ip4_fib_counter_t", "counter", 0, "length"]
 
         ]'''
 
@@ -202,21 +201,8 @@ class TestPAPI(unittest.TestCase):
         self.assertEqual(1235, rv.counter[0].packets)
         self.assertEqual(333, rv.counter[1].packets)
 
-
-    def test_compound_data_type(self):
-        # Normal field
-        # Fixed array
-        # Variable array
-        pass
-
-    def test_return_tuple(self):
-        # Normal field
-        # Fixed array
-        # Variable array
-        pass
-
     def test_simple_array(self):
-        msglist = VPP([])
+        msglist = VPP(testmode=True)
 
         simple_byte_array = '''["simple_byte_array",
             ["u32", "length"],
@@ -268,8 +254,98 @@ class TestPAPI(unittest.TestCase):
         self.assertEqual(6, rv.length)
         self.assertEqual('foobar', rv.list)
 
+    def test_old_vla_array(self):
+        msglist = VPP(testmode = True)
+
+        # VLA
+        vla_byte_array = '''["vla_byte_array",
+            ["u32", "foobar"],
+            ["u32", "list", 2],
+            ["u32", "propercount"],
+            ["u8", "propermask", 0, "propercount"],
+            ["u8", "oldmask", 0],
+            {"crc" : "0xb2739495"}
+        ]'''
+        p = json.loads(vla_byte_array)
+        msgdef = msglist.add_message(p[0], p[1:])
+        b = msglist.encode(msgdef, {'list' : [123, 456], 'oldmask': b'foobar',
+                                    'propercount' : 2,
+                                    'propermask' : [8,9]})
+        self.assertEqual(24, len(b))
+        rv = msglist.decode(msgdef, b)
+        self.assertEqual(b'foobar', rv.oldmask)
+
+    def test_old_vla_array_not_last_member(self):
+        msglist = VPP(testmode = True)
+
+        # VLA
+        vla_byte_array = '''["vla_byte_array",
+            ["u8", "oldmask", 0],
+            ["u32", "foobar"],
+            {"crc" : "0xb2739495"}
+        ]'''
+        p = json.loads(vla_byte_array)
+        self.assertRaises(ValueError, msglist.add_message, p[0], p[1:])
+
+    def test_old_vla_array_u32(self):
+        msglist = VPP(testmode = True)
+
+        # VLA
+        vla_byte_array = '''["vla_byte_array",
+            ["u32", "foobar"],
+            ["u32", "oldmask", 0],
+            {"crc" : "0xb2739495"}
+        ]'''
+        p = json.loads(vla_byte_array)
+        msgdef = msglist.add_message(p[0], p[1:])
+        b = msglist.encode(msgdef, {'foobar' : 123, 'oldmask': [123, 456, 789]})
+        self.assertEqual(16, len(b))
+        rv = msglist.decode(msgdef, b)
+        self.assertEqual([123, 456, 789], rv.oldmask)
+
+    def test_old_vla_array_compound(self):
+        msglist = VPP(testmode = True)
+
+        # VLA
+        counter_type = '''["ip4_fib_counter",
+            ["u32", "address"],
+            ["u8", "address_length"],
+            ["u64", "packets"],
+            ["u64", "bytes"],
+            {"crc" : "0xb2739495"}
+        ]'''
+
+        vla_byte_array = '''["vla_byte_array",
+            ["vl_api_ip4_fib_counter_t", "counter", 0],
+            {"crc" : "0xb2739495"}
+        ]'''
+
+        p = json.loads(counter_type)
+        msglist.add_type(p[0], p[1:])
+
+        p = json.loads(vla_byte_array)
+        with self.assertRaises(NotImplementedError):
+            msgdef = msglist.add_message(p[0], p[1:])
+
+    def test_array_count_not_previous(self):
+        msglist = VPP(testmode = True)
+
+        # VLA
+        vla_byte_array = '''["vla_byte_array",
+            ["u32", "count"],
+            ["u32", "filler"],
+            ["u32", "lst", 0, "count"],
+            {"crc" : "0xb2739495"}
+        ]'''
+
+        p = json.loads(vla_byte_array)
+        msgdef = msglist.add_message(p[0], p[1:])
+        b = msglist.encode(msgdef, {'count': 3, 'lst': [1,2,3], 'filler' : 1 })
+        rv = msglist.decode(msgdef, b)
+        self.assertEqual(rv.lst, [1,2,3])
+
     def test_argument_name(self):
-        msglist = VPP([])
+        msglist = VPP(testmode=True)
 
 
         simple_name = '''["simple_name",
@@ -292,7 +368,7 @@ class TestConnectedPAPI(unittest.TestCase):
 
         rv = vpp.show_version()
         self.assertEqual(0, rv.retval)
-        print('RV', rv.program.decode().rstrip('\0x00'))
+        self.assertEqual('vpe', rv.program.decode().rstrip('\0x00'))
         vpp.disconnect()
 
 
diff --git a/vpp-api/python/vpp_papi/vpp_papi.py b/vpp-api/python/vpp_papi/vpp_papi.py
index 8e7b7f66ff5..02fe2457ef0 100644
--- a/vpp-api/python/vpp_papi/vpp_papi.py
+++ b/vpp-api/python/vpp_papi/vpp_papi.py
@@ -15,7 +15,7 @@
 #
 
 from __future__ import print_function
-import sys, os, logging, collections, struct, json, threading
+import sys, os, logging, collections, struct, json, threading, glob
 logging.basicConfig(level=logging.DEBUG)
 import vpp_api
 
@@ -23,7 +23,7 @@ def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
 
 class VPP():
-    def __init__(self, apifiles):
+    def __init__(self, apifiles = None, testmode = False):
         self.messages = {}
         self.id_names = []
         self.id_msgdef = []
@@ -34,6 +34,10 @@ class VPP():
         self.timeout = 5
         self.apifile = []
 
+        if not apifiles:
+            # Pick up API definitions from default directory
+            apifiles = glob.glob('/usr/share/vpp/api/*.api.json')
+
         for file in apifiles:
             self.apifile.append(file)
             with open(file) as apidef_file:
@@ -45,8 +49,8 @@ class VPP():
                     self.add_message(m[0], m[1:])
 
         # Basic sanity check
-        if not 'control_ping' in self.messages:
-            raise ValueError(1, 'Error in JSON message definitions')
+        if len(self.messages) == 0 and not testmode:
+            raise ValueError(1, 'Missing JSON message definitions')
 
 
     class ContextId(object):
@@ -61,7 +65,7 @@ class VPP():
         print('Connected') if self.connected else print('Not Connected')
         print('Read API definitions from', self.apifile)
 
-    def __struct (self, t, n = None, e = 0, vl = None):
+    def __struct (self, t, n = None, e = -1, vl = None):
         base_types = { 'u8' : 'B',
                        'u16' : 'H',
                        'u32' : 'I',
@@ -73,33 +77,40 @@ class VPP():
         if t in base_types:
             pack = base_types[t]
             if not vl:
-                if e and t == 'u8':
+                if e > 0 and t == 'u8':
                     # Fixed byte array
                     return struct.Struct('>' + str(e) + 's')
-                if e:
+                if e > 0:
                     # Fixed array of base type
                     return [e, struct.Struct('>' + base_types[t])]
+                elif e == 0:
+                    # Old style variable array
+                    return [-1, struct.Struct('>' + base_types[t])]
             else:
                 # Variable length array
-                return [vl, struct.Struct('>s')] if t == 'u8' else [vl, struct.Struct('>' + base_types[t])]
+                return [vl, struct.Struct('>s')] if t == 'u8' else \
+                    [vl, struct.Struct('>' + base_types[t])]
 
             return struct.Struct('>' + base_types[t])
 
         if t in self.messages:
             ### Return a list in case of array ###
-            if e and not vl:
+            if e > 0 and not vl:
                 return [e, lambda self, encode, buf, offset, args: (
-                    self.__struct_type(encode, self.messages[t], buf, offset, args)
-                )]
+                    self.__struct_type(encode, self.messages[t], buf, offset,
+                                       args))]
             if vl:
                 return [vl, lambda self, encode, buf, offset, args: (
-                    self.__struct_type(encode, self.messages[t], buf, offset, args)
-                )]
+                    self.__struct_type(encode, self.messages[t], buf, offset,
+                                       args))]
+            elif e == 0:
+                # Old style VLA
+                raise NotImplementedError(1, 'No support for compound types ' + t)
             return lambda self, encode, buf, offset, args: (
                 self.__struct_type(encode, self.messages[t], buf, offset, args)
             )
 
-        raise ValueError
+        raise ValueError(1, 'Invalid message type: ' + t)
 
     def __struct_type(self, encode, msgdef, buf, offset, kwargs):
         if encode:
@@ -120,10 +131,7 @@ class VPP():
             if k in kwargs:
                 if type(v) is list:
                     if callable(v[1]):
-                        if v[0] in kwargs:
-                            e = kwargs[v[0]]
-                        else:
-                            e = v[0]
+                        e = kwargs[v[0]] if v[0] in kwargs else v[0]
                         size = 0
                         for i in range(e):
                             size += v[1](self, True, buf, off + size,
@@ -179,7 +187,7 @@ class VPP():
                 if callable(v[1]): # compound type
                     size = 0
                     if v[0] in msgdef['args']: # vla
-                        e = int(res[-1])
+                        e = res[v[2]]
                     else: # fixed array
                         e = v[0]
                     res.append(lst)
@@ -188,14 +196,19 @@ class VPP():
                         lst.append(l)
                         size += s
                     continue
-
                 if v[1].size == 1:
-                    size = int(res[-1])
+                    if type(v[0]) is int:
+                        size = len(buf) - off
+                    else:
+                        size = res[v[2]]
                     res.append(buf[off:off + size])
                 else:
-                    e = int(res[-1])
+                    e = v[0] if type(v[0]) is int else res[v[2]]
+                    if e == -1:
+                        e = (len(buf) - off) / v[1].size
                     if e == 0:
-                        raise ValueError
+                        raise ValueError(1,
+                                         'Variable length array, empty length: ' + k)
                     lst = []
                     res.append(lst)
                     size = 0
@@ -226,14 +239,18 @@ class VPP():
         argtypes = collections.OrderedDict()
         fields = []
         msg = {}
-        for f in msgdef:
+        for i, f in enumerate(msgdef):
             if type(f) is dict and 'crc' in f:
                 msg['crc'] = f['crc']
                 continue
             field_type = f[0]
             field_name = f[1]
+            if len(f) == 3 and f[2] == 0 and i != len(msgdef) - 2:
+                raise ValueError('Variable Length Array must be last: ' + name)
             args[field_name] = self.__struct(*f)
             argtypes[field_name] = field_type
+            if len(f) == 4: # Find offset to # elements field
+                args[field_name].append(args.keys().index(f[3]) - i)
             fields.append(field_name)
         msg['return_tuple'] = collections.namedtuple(name, fields,
                                                      rename = True)
@@ -243,7 +260,7 @@ class VPP():
         return self.messages[name]
 
     def add_type(self, name, typedef):
-        self.add_message('vl_api_' + name + '_t', typedef)
+        return self.add_message('vl_api_' + name + '_t', typedef)
 
     def make_function(self, name, i, msgdef, multipart, async):
         if (async):
diff --git a/vpp/vpp-api/vpe.api b/vpp/vpp-api/vpe.api
index b13c2606667..d5818cb7518 100644
--- a/vpp/vpp-api/vpe.api
+++ b/vpp/vpp-api/vpe.api
@@ -814,8 +814,8 @@ define sr_tunnel_add_del
   u16 flags_net_byte_order;
   u8 n_segments;
   u8 n_tags;
-  u8 segs_and_tags[0];
   u8 policy_name[64];
+  u8 segs_and_tags[0];
 };
 
 /** \brief IPv6 segment routing tunnel add / del response