VPPAPIGEN: Add union and enum support and IP4/IP6 address type.

Note: The Python, Java and C/C++ bindings must be updated before ip/ip_types.api can be used.

ip_types.api:

typedef ip4_address {
  u8 address[4];
};

typedef ip6_address {
  u8 address[16];
};

enum address_family {
  ADDRESS_IP4 = 0,
  ADDRESS_IP6,
};

union address_union {
  vl_api_ip4_address_t ip4;
  vl_api_ip6_address_t ip6;
};

typedef address {
  vl_api_address_family_t af;
  vl_api_address_union_t un;
};

Change-Id: I22f67092f24db5bd650a03c6f446a84cd9fd1074
Signed-off-by: Ole Troan <ot@cisco.com>

3 files changed, 123 insertions, 85 deletions

diff --git a/src/tools/vppapigen/C.py b/src/tools/vppapigen/C.py
index 2a8930666fa..c9e543f486b 100644
--- a/src/tools/vppapigen/C.py
+++ b/src/tools/vppapigen/C.py
@@ -42,7 +42,7 @@ def msg_ids(s):
 #ifdef vl_msg_id
 '''
 
-    for t in s['defines']:
+    for t in s['Define']:
         output += "vl_msg_id(VL_API_%s, vl_api_%s_t_handler)\n" % \
                   (t.name.upper(), t.name)
     output += "#endif"
@@ -58,7 +58,7 @@ def msg_names(s):
 #ifdef vl_msg_name
 '''
 
-    for t in s['defines']:
+    for t in s['Define']:
         dont_trace = 0 if t.dont_trace else 1
         output += "vl_msg_name(vl_api_%s_t, %d)\n" % (t.name, dont_trace)
     output += "#endif"
@@ -75,7 +75,7 @@ def msg_name_crc_list(s, suffix):
 '''
     output += "#define foreach_vl_msg_name_crc_%s " % suffix
 
-    for t in s['defines']:
+    for t in s['Define']:
         output += "\\\n_(VL_API_%s, %s, %08x) " % \
                    (t.name.upper(), t.name, t.crc)
     output += "\n#endif"
@@ -93,7 +93,7 @@ def duplicate_wrapper_tail():
     return '#endif\n\n'
 
 
-def typedefs(s, filename):
+def typedefs(objs, filename):
     name = filename.replace('.', '_')
     output = '''\
 
@@ -105,30 +105,32 @@ def typedefs(s, filename):
 #define included_{module}
 '''
     output = output.format(module=name)
-    for e in s['enums']:
-        output += duplicate_wrapper_head(e.name)
-        output += "typedef enum {\n"
-        for b in e.block:
-            output += "    %s = %s,\n" % (b[0], b[1])
-        output += '} vl_api_%s_t;\n' % e.name
-        output += duplicate_wrapper_tail()
-
-    for t in s['typedefs'] + s['defines']:
-        output += duplicate_wrapper_head(t.name)
-        output += "typedef VL_API_PACKED(struct _vl_api_%s {\n" % t.name
-        for b in t.block:
-            if b.type == 'Field':
-                output += "    %s %s;\n" % (b.fieldtype, b.fieldname)
-            elif b.type == 'Array':
-                if b.lengthfield:
-                    output += "    %s %s[0];\n" % (b.fieldtype, b.fieldname)
-                else:
-                    output += "    %s %s[%s];\n" % (b.fieldtype, b.fieldname,
-                                                    b.length)
+    for o in objs:
+        tname = o.__class__.__name__
+        output += duplicate_wrapper_head(o.name)
+        if tname == 'Enum':
+            output += "typedef enum {\n"
+            for b in o.block:
+                output += "    %s = %s,\n" % (b[0], b[1])
+            output += '} vl_api_%s_t;\n' % o.name
+        else:
+            if tname == 'Union':
+                output += "typedef VL_API_PACKED(union _vl_api_%s {\n" % o.name
             else:
-                raise ValueError("Error in processing array type %s" % b)
+                output += "typedef VL_API_PACKED(struct _vl_api_%s {\n" % o.name
+            for b in o.block:
+                if b.type == 'Field':
+                    output += "    %s %s;\n" % (b.fieldtype, b.fieldname)
+                elif b.type == 'Array':
+                    if b.lengthfield:
+                        output += "    %s %s[0];\n" % (b.fieldtype, b.fieldname)
+                    else:
+                        output += "    %s %s[%s];\n" % (b.fieldtype, b.fieldname,
+                                                        b.length)
+                else:
+                    raise ValueError("Error in processing array type %s" % b)
 
-        output += '}) vl_api_%s_t;\n' % t.name
+            output += '}) vl_api_%s_t;\n' % o.name
         output += duplicate_wrapper_tail()
 
     output += "\n#endif"
@@ -148,7 +150,7 @@ format_strings = {'u8': '%u',
                   'f64': '%.2f', }
 
 
-def printfun(s):
+def printfun(objs):
     output = '''\
 /****** Print functions *****/
 #ifdef vl_printfun
@@ -162,7 +164,9 @@ def printfun(s):
 #endif
 
 '''
-    for t in s['typedefs'] + s['defines']:
+    for t in objs:
+        if t.__class__.__name__ == 'Enum':
+            continue
         if t.manual_print:
             output += "/***** manual: vl_api_%s_t_print  *****/\n\n" % t.name
             continue
@@ -199,7 +203,7 @@ endian_strings = {
 }
 
 
-def endianfun(s):
+def endianfun(objs):
     output = '''\
 
 /****** Endian swap functions *****/\n\
@@ -214,7 +218,9 @@ def endianfun(s):
 
 '''
 
-    for t in s['typedefs'] + s['defines']:
+    for t in objs:
+        if t.__class__.__name__ == 'Enum':
+            continue
         if t.manual_endian:
             output += "/***** manual: vl_api_%s_t_endian  *****/\n\n" % t.name
             continue
@@ -247,8 +253,8 @@ def version_tuple(s, module):
 #ifdef vl_api_version_tuple
 
 '''
-    if 'version' in s['options']:
-        v = s['options']['version']
+    if 'version' in s['Option']:
+        v = s['Option']['version']
         (major, minor, patch) = v.split('.')
         output += "vl_api_version_tuple(%s, %s, %s, %s)\n" % \
                   (module, major, minor, patch)
@@ -269,9 +275,9 @@ def run(input_filename, s, file_crc):
     output += msg_ids(s)
     output += msg_names(s)
     output += msg_name_crc_list(s, filename)
-    output += typedefs(s, filename + file_extension)
-    output += printfun(s)
-    output += endianfun(s)
+    output += typedefs(s['types'] + s['Define'], filename + file_extension)
+    output += printfun(s['types'] + s['Define'])
+    output += endianfun(s['types'] + s['Define'])
     output += version_tuple(s, basename)
     output += bottom_boilerplate.format(input_filename=basename,
                                         file_crc=file_crc)
diff --git a/src/tools/vppapigen/JSON.py b/src/tools/vppapigen/JSON.py
index 73bff76cbb7..2991bec57ac 100644
--- a/src/tools/vppapigen/JSON.py
+++ b/src/tools/vppapigen/JSON.py
@@ -40,6 +40,8 @@ def walk_defs(s):
                     f = [b.fieldtype, b.fieldname, b.length, b.lengthfield]
                 else:
                     f = [b.fieldtype, b.fieldname, b.length]
+            elif b.type == 'Union':
+                print('UNION')
             else:
                 raise ValueError("Error in processing array type %s" % b)
             d.append(f)
@@ -58,9 +60,10 @@ def walk_defs(s):
 def run(filename, s, file_crc):
     j = {}
 
-    j['types'] = walk_defs(s['typedefs'])
-    j['messages'] = walk_defs(s['defines'])
-    j['enums'] = walk_enums(s['enums'])
-    j['services'] = walk_services(s['services'])
+    j['types'] = walk_defs([o for o in s['types'] if o.__class__.__name__ == 'Typedef'])
+    j['messages'] = walk_defs(s['Define'])
+    j['unions'] = walk_defs([o for o in s['types'] if o.__class__.__name__ == 'Union'])
+    j['enums'] = walk_enums([o for o in s['types'] if o.__class__.__name__ == 'Enum'])
+    j['services'] = walk_services(s['Service'])
     j['vl_api_version'] = hex(file_crc)
     return json.dumps(j, indent=4, separators=(',', ': '))
diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py
index 2e8d80f9d0c..5cedfb214e5 100755
--- a/src/tools/vppapigen/vppapigen.py
+++ b/src/tools/vppapigen/vppapigen.py
@@ -71,6 +71,7 @@ class VPPAPILexer(object):
         'import': 'IMPORT',
         'true': 'TRUE',
         'false': 'FALSE',
+        'union': 'UNION',
     }
 
     tokens = ['STRING_LITERAL',
@@ -137,12 +138,35 @@ class Typedef():
         self.block = block
         self.crc = binascii.crc32(str(block)) & 0xffffffff
         global_crc = binascii.crc32(str(block), global_crc)
+        self.manual_print = False
+        self.manual_endian = False
+        for f in flags:
+            if f == 'manual_print':
+                self.manual_print = True
+            elif f == 'manual_endian':
+                self.manual_endian = True
         global_type_add(name)
 
     def __repr__(self):
         return self.name + str(self.flags) + str(self.block)
 
 
+class Union():
+    def __init__(self, name, block):
+        self.type = 'Union'
+        self.manual_print = False
+        self.manual_endian = False
+        global global_crc
+        self.name = name
+        self.block = block
+        self.crc = binascii.crc32(str(block)) & 0xffffffff
+        global_crc = binascii.crc32(str(block), global_crc)
+        global_type_add(name)
+
+    def __repr__(self):
+        return str(self.block)
+
+
 class Define():
     def __init__(self, name, flags, block):
         global global_crc
@@ -151,17 +175,13 @@ class Define():
         self.block = block
         self.crc = binascii.crc32(str(block)) & 0xffffffff
         global_crc = binascii.crc32(str(block), global_crc)
-        self.typeonly = False
         self.dont_trace = False
         self.manual_print = False
         self.manual_endian = False
         self.autoreply = False
         self.singular = False
         for f in flags:
-            if f == 'typeonly':
-                self.typeonly = True
-                global_type_add(name)
-            elif f == 'dont_trace':
+            if f == 'dont_trace':
                 self.dont_trace = True
             elif f == 'manual_print':
                 self.manual_print = True
@@ -185,6 +205,7 @@ class Enum():
         global global_crc
         self.name = name
         self.enumtype = enumtype
+
         count = 0
         for i, b in enumerate(block):
             if type(b) is list:
@@ -334,6 +355,7 @@ class VPPAPIParser(object):
                 | option
                 | import
                 | enum
+                | union
                 | service'''
         p[0] = p[1]
 
@@ -407,7 +429,11 @@ class VPPAPIParser(object):
 
     def p_define_flist(self, p):
         '''define : flist DEFINE ID '{' block_statements_opt '}' ';' '''
-        p[0] = Define(p[3], p[1], p[5])
+        # Legacy typedef
+        if 'typeonly' in p[1]:
+            p[0] = Typedef(p[3], p[1], p[5])
+        else:
+            p[0] = Define(p[3], p[1], p[5])
 
     def p_flist(self, p):
         '''flist : flag
@@ -432,7 +458,7 @@ class VPPAPIParser(object):
         p[0] = Typedef(p[2], [], p[4])
 
     def p_block_statements_opt(self, p):
-        '''block_statements_opt : block_statements'''
+        '''block_statements_opt : block_statements '''
         p[0] = p[1]
 
     def p_block_statements(self, p):
@@ -526,6 +552,10 @@ class VPPAPIParser(object):
                               self._token_coord(p, 1))
         p[0] = p[1]
 
+    def p_union(self, p):
+        '''union : UNION ID '{' block_statements_opt '}' ';' '''
+        p[0] = Union(p[2], p[4])
+
     # Error rule for syntax errors
     def p_error(self, p):
         if p:
@@ -559,35 +589,33 @@ class VPPAPI(object):
 
     def process(self, objs):
         s = {}
-        s['defines'] = []
-        s['typedefs'] = []
-        s['imports'] = []
-        s['options'] = {}
-        s['enums'] = []
-        s['services'] = []
-
+        s['Option'] = {}
+        s['Define'] = []
+        s['Service'] = []
+        s['types'] = []
+        s['Import'] = []
         for o in objs:
+            tname = o.__class__.__name__
             if isinstance(o, Define):
-                if o.typeonly:
-                    s['typedefs'].append(o)
-                else:
-                    s['defines'].append(o)
-                    if o.autoreply:
-                        s['defines'].append(self.autoreply_block(o.name))
+                s[tname].append(o)
+                if o.autoreply:
+                    s[tname].append(self.autoreply_block(o.name))
             elif isinstance(o, Option):
-                s['options'][o[1]] = o[2]
-            elif isinstance(o, Enum):
-                s['enums'].append(o)
-            elif isinstance(o, Typedef):
-                s['typedefs'].append(o)
+                s[tname][o[1]] = o[2]
             elif type(o) is list:
                 for o2 in o:
                     if isinstance(o2, Service):
-                        s['services'].append(o2)
+                        s['Service'].append(o2)
+            elif isinstance(o, Enum) or isinstance(o, Typedef) or isinstance(o, Union):
+                s['types'].append(o)
+            else:
+                if tname not in s:
+                    raise ValueError('Unknown class type: {} {}'.format(tname, o))
+                s[tname].append(o)
 
-        msgs = {d.name: d for d in s['defines']}
-        svcs = {s.caller: s for s in s['services']}
-        replies = {s.reply: s for s in s['services']}
+        msgs = {d.name: d for d in s['Define']}
+        svcs = {s.caller: s for s in s['Service']}
+        replies = {s.reply: s for s in s['Service']}
         seen_services = {}
 
         for service in svcs:
@@ -627,7 +655,7 @@ class VPPAPI(object):
                 if d in svcs:
                     continue
                 if d[:-5]+'_details' in msgs:
-                    s['services'].append(Service(d, d[:-5]+'_details',
+                    s['Service'].append(Service(d, d[:-5]+'_details',
                                                  stream=True))
                 else:
                     raise ValueError('{} missing details message'
@@ -643,7 +671,7 @@ class VPPAPI(object):
             if d in svcs:
                 continue
             if d+'_reply' in msgs:
-                s['services'].append(Service(d, d+'_reply'))
+                s['Service'].append(Service(d, d+'_reply'))
             else:
                 raise ValueError(
                     '{} missing reply message ({}) or service definition'
@@ -651,18 +679,18 @@ class VPPAPI(object):
 
         return s
 
-    def process_imports(self, objs, in_import):
+    def process_imports(self, objs, in_import, result):
         imported_objs = []
         for o in objs:
-            if isinstance(o, Import):
-                return self.process_imports(o.result, True) + objs
-            if in_import:
-                if isinstance(o, Define) and o.typeonly:
-                    imported_objs.append(o)
-        if in_import:
-            return imported_objs
-        return objs
+            # Only allow the following object types from imported file
+            if in_import and not (isinstance(o, Enum) or
+                                  isinstance(o, Union) or
+                                  isinstance(o, Typedef)):
+                continue
+            result.append(o)
 
+            if isinstance(o, Import):
+                self.process_imports(o.result, True, result)
 
 # Add message ids to each message.
 def add_msg_id(s):
@@ -720,14 +748,15 @@ def main():
     log = logging.getLogger('vppapigen')
 
     parser = VPPAPI(debug=args.debug, filename=filename, logger=log)
-    result = parser.parse_file(args.input, log)
+    parsed_objects = parser.parse_file(args.input, log)
 
     # Build a list of objects. Hash of lists.
-    result = parser.process_imports(result, False)
+    result = []
+    parser.process_imports(parsed_objects, False, result)
     s = parser.process(result)
 
     # Add msg_id field
-    s['defines'] = add_msg_id(s['defines'])
+    s['Define'] = add_msg_id(s['Define'])
 
     file_crc = global_crc & 0xffffffff
 
@@ -736,10 +765,10 @@ def main():
     if args.debug:
         import pprint
         pp = pprint.PrettyPrinter(indent=4)
-        for t in s['defines']:
-            pp.pprint([t.name, t.flags, t.block])
-        for t in s['typedefs']:
+        for t in s['Define']:
             pp.pprint([t.name, t.flags, t.block])
+        for t in s['types']:
+            pp.pprint([t.name, t.block])
 
     #
     # Generate representation