vppapigen: add parser support for enumflags

Type: improvement

Change-Id: I0f15862cc8399a4f7c8a81fe44ba8b27d8772278
Signed-off-by: Paul Vinciguerra <pvinci@vinciconsulting.com>
Signed-off-by: Ole Troan <ot@cisco.com>
(cherry picked from commit e15523297bb3905f2e0eef4272fc69a8a92463cc)

4 files changed, 221 insertions, 57 deletions

diff --git a/src/tools/vppapigen/test_vppapigen.py b/src/tools/vppapigen/test_vppapigen.py
index 999addfccd2..c454ffc8638 100755
--- a/src/tools/vppapigen/test_vppapigen.py
+++ b/src/tools/vppapigen/test_vppapigen.py
@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 
 import unittest
-from vppapigen import VPPAPI, Option, ParseError, Union, foldup_crcs, global_types
+from vppapigen import VPPAPI, Option, ParseError, Union, foldup_crcs, \
+    global_types
 import vppapigen
 
+
 # TODO
 # - test parsing of options, typedefs, enums, defines
 # - test JSON, C output
@@ -19,6 +21,7 @@ class TestVersion(unittest.TestCase):
         r = self.parser.parse_string(version_string)
         self.assertTrue(isinstance(r[0], Option))
 
+
 class TestUnion(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -49,7 +52,6 @@ class TestUnion(unittest.TestCase):
         self.assertTrue(r[0].vla)
         s = self.parser.process(r)
 
-
         test_string2 = '''
         union foo_union_vla2 {
         u32 a;
@@ -74,6 +76,7 @@ class TestUnion(unittest.TestCase):
         '''
         self.assertRaises(ValueError, self.parser.parse_string, test_string3)
 
+
 class TestTypedef(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -169,7 +172,7 @@ class TestCRC(unittest.TestCase):
         '''
         crc = get_crc(test_string, 'foo')
 
-        # modify underlaying type
+        # modify underlying type
         test_string = '''
          typedef list { u8 foo2; };
          autoreply define foo { u8 foo;  vl_api_list_t l;};
@@ -255,5 +258,97 @@ autoreply define sr_policy_add
 
         self.assertNotEqual(crc, crc2)
 
+
+class TestEnum(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = VPPAPI()
+
+    def test_enum_as_enum(self):
+        test_string = """\
+enum tunnel_mode : u8
+{
+  /** point-to-point */
+  TUNNEL_API_MODE_P2P = 0,
+  /** multi-point */
+  TUNNEL_API_MODE_MP,
+};
+"""
+        r = self.parser.parse_string(test_string)
+        self.assertIsNotNone(r)
+        s = self.parser.process(r)
+        for o in s['types']:
+            if o.type == 'Enum':
+                self.assertEqual(o.name, "tunnel_mode")
+                break
+        else:
+            self.fail()
+
+    def test_enumflag_as_enum(self):
+        test_string = """\
+enum virtio_flags {
+        VIRTIO_API_FLAG_GSO = 1, /* enable gso on the interface */
+        VIRTIO_API_FLAG_CSUM_OFFLOAD = 2, /* enable checksum offload without gso on the interface */
+        VIRTIO_API_FLAG_GRO_COALESCE = 4, /* enable packet coalescing on tx side, provided gso enabled */
+        VIRTIO_API_FLAG_PACKED = 8, /* enable packed ring support, provided it is available from backend */
+        VIRTIO_API_FLAG_IN_ORDER = 16, /* enable in order support, provided it is available from backend */
+        VIRTIO_API_FLAG_BUFFERING = 32 [backwards_compatible], /* enable buffering to handle backend jitter/delays */
+};"""
+        r = self.parser.parse_string(test_string)
+        self.assertIsNotNone(r)
+        s = self.parser.process(r)
+        for o in s['types']:
+            if o.type == 'Enum':
+                self.assertEqual(o.name, "virtio_flags")
+                break
+        else:
+            self.fail()
+
+
+class TestEnumFlag(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = VPPAPI()
+
+    def test_enum_as_enumflag(self):
+        test_string = """\
+enumflag tunnel_mode_ef : u8
+{
+  /** point-to-point */
+  TUNNEL_API_MODE_P2P = 0,
+  /** multi-point */
+  TUNNEL_API_MODE_MP,
+  TUNNEL_API_MODE_FOO,
+  TUNNEL_API_MODE_BAR,
+};"""
+        with self.assertRaises(TypeError) as ctx:
+            r = self.parser.parse_string(test_string)
+
+        self.assertTrue(str(ctx.exception).startswith(
+            'tunnel_mode_ef is not a flag enum.'))
+
+    def test_enumflag_as_enumflag(self):
+        test_string = """\
+enumflag virtio_flags_ef {
+        VIRTIO_API_FLAG_GSO = 1, /* enable gso on the interface */
+        VIRTIO_API_FLAG_CSUM_OFFLOAD = 2, /* enable checksum offload without gso on the interface */
+        VIRTIO_API_FLAG_GRO_COALESCE = 4, /* enable packet coalescing on tx side, provided gso enabled */
+        VIRTIO_API_FLAG_PACKED = 8, /* enable packed ring support, provided it is available from backend */
+        VIRTIO_API_FLAG_IN_ORDER = 16, /* enable in order support, provided it is available from backend */
+        VIRTIO_API_FLAG_BUFFERING = 32 [backwards_compatible], /* enable buffering to handle backend jitter/delays */
+};"""
+        r = self.parser.parse_string(test_string)
+        self.assertIsNotNone(r)
+        s = self.parser.process(r)
+        for o in s['types']:
+            if o.type == 'EnumFlag':
+                self.assertEqual(o.name, "virtio_flags_ef")
+                break
+        else:
+            self.fail()
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py
index da008231a32..b80dd4d9f7d 100755
--- a/src/tools/vppapigen/vppapigen.py
+++ b/src/tools/vppapigen/vppapigen.py
@@ -58,6 +58,7 @@ class VPPAPILexer(object):
         'define': 'DEFINE',
         'typedef': 'TYPEDEF',
         'enum': 'ENUM',
+        'enumflag': 'ENUMFLAG',
         'typeonly': 'TYPEONLY',
         'manual_print': 'MANUAL_PRINT',
         'manual_endian': 'MANUAL_ENDIAN',
@@ -182,7 +183,16 @@ def vla_is_last_check(name, block):
     return vla
 
 
-class Service():
+class Processable:
+    type = "<Invalid>"
+
+    def process(self, result):  # -> Dict
+        result[self.type].append(self)
+
+
+class Service(Processable):
+    type = 'Service'
+
     def __init__(self, caller, reply, events=None, stream_message=None,
                  stream=False):
         self.caller = caller
@@ -192,10 +202,11 @@ class Service():
         self.events = [] if events is None else events
 
 
-class Typedef():
+class Typedef(Processable):
+    type = 'Typedef'
+
     def __init__(self, name, flags, block):
         self.name = name
-        self.type = 'Typedef'
         self.flags = flags
         self.block = block
         self.crc = str(block).encode()
@@ -211,14 +222,18 @@ class Typedef():
         self.vla = vla_is_last_check(name, block)
         vla_mark_length_field(self.block)
 
+    def process(self, result):
+        result['types'].append(self)
+
     def __repr__(self):
         return self.name + str(self.flags) + str(self.block)
 
 
-class Using():
+class Using(Processable):
+    type = 'Using'
+
     def __init__(self, name, flags, alias):
         self.name = name
-        self.type = 'Using'
         self.vla = False
         self.block = []
         self.manual_print = True
@@ -248,13 +263,17 @@ class Using():
         self.crc = str(self.block).encode()
         global_type_add(name, self)
 
+    def process(self, result):  # -> Dict
+        result['types'].append(self)
+
     def __repr__(self):
         return self.name + str(self.alias)
 
 
-class Union():
+class Union(Processable):
+    type = 'Union'
+
     def __init__(self, name, flags, block):
-        self.type = 'Union'
         self.manual_print = False
         self.manual_endian = False
         self.name = name
@@ -271,14 +290,18 @@ class Union():
 
         global_type_add(name, self)
 
+    def process(self, result):
+        result['types'].append(self)
+
     def __repr__(self):
         return str(self.block)
 
 
-class Define():
+class Define(Processable):
+    type = 'Define'
+
     def __init__(self, name, flags, block):
         self.name = name
-        self.type = 'Define'
         self.flags = flags
         self.block = block
         self.dont_trace = False
@@ -313,16 +336,31 @@ class Define():
 
         self.crc = str(block).encode()
 
+    def autoreply_block(self, name, parent):
+        block = [Field('u32', 'context'),
+                 Field('i32', 'retval')]
+        # inherit the parent's options
+        for k, v in parent.options.items():
+            block.append(Option(k, v))
+        return Define(name + '_reply', [], block)
+
+    def process(self, result):  # -> Dict
+        tname = self.__class__.__name__
+        result[tname].append(self)
+        if self.autoreply:
+            result[tname].append(self.autoreply_block(self.name, self))
+
     def __repr__(self):
         return self.name + str(self.flags) + str(self.block)
 
 
-class Enum():
+class Enum(Processable):
+    type = 'Enum'
+
     def __init__(self, name, block, enumtype='u32'):
         self.name = name
         self.enumtype = enumtype
         self.vla = False
-        self.type = 'Enum'
         self.manual_print = False
 
         count = 0
@@ -350,12 +388,30 @@ class Enum():
         self.crc = str(block3).encode()
         global_type_add(name, self)
 
+    def process(self, result):
+        result['types'].append(self)
+
     def __repr__(self):
         return self.name + str(self.block)
 
 
-class Import():
+class EnumFlag(Enum):
+    type = 'EnumFlag'
+
+    def __init__(self, name, block, enumtype='u32'):
+        super(EnumFlag, self).__init__(name, block, enumtype)
+
+        for b in self.block:
+            if bin(b[1])[2:].count("1") > 1:
+                raise TypeError("%s is not a flag enum.  No element in a "
+                                "flag enum may have more than a "
+                                "single bit set." % self.name)
+
+
+class Import(Processable):
+    type = 'Import'
     _initialized = False
+
     def __new__(cls, *args, **kwargs):
         if args[0] not in seen_imports:
             instance = super().__new__(cls)
@@ -383,13 +439,17 @@ class Import():
         return self.filename
 
 
-class Option():
+class Option(Processable):
+    type = 'Option'
+
     def __init__(self, option, value=None):
-        self.type = 'Option'
         self.option = option
         self.value = value
         self.crc = str(option).encode()
 
+    def process(self, result):  # -> Dict
+        result[self.type][self.option] = self.value
+
     def __repr__(self):
         return str(self.option)
 
@@ -397,9 +457,10 @@ class Option():
         return self.option[index]
 
 
-class Array():
+class Array(Processable):
+    type = 'Array'
+
     def __init__(self, fieldtype, name, length, modern_vla=False):
-        self.type = 'Array'
         self.fieldtype = fieldtype
         self.fieldname = name
         self.modern_vla = modern_vla
@@ -417,9 +478,12 @@ class Array():
                     self.lengthfield])
 
 
-class Field():
+class Field(Processable):
+    type = 'Field'
+
     def __init__(self, fieldtype, name, limit=None):
-        self.type = 'Field'
+        # limit field has been expanded to an options dict.
+
         self.fieldtype = fieldtype
         self.is_lengthfield = False
 
@@ -437,18 +501,28 @@ class Field():
         return str([self.fieldtype, self.fieldname])
 
 
-class Counter():
+class Counter(Processable):
+    type = 'Counter'
+
     def __init__(self, path, counter):
-        self.type = 'Counter'
         self.name = path
         self.block = counter
 
+    def process(self, result):  # -> Dict
+        result['Counters'].append(self)
+
+
+class Paths(Processable):
+    type = 'Paths'
 
-class Paths():
     def __init__(self, pathset):
-        self.type = 'Paths'
         self.paths = pathset
 
+    def __repr__(self):
+        return "%s(paths=%s)" % (
+            self.__class__.__name__, self.paths
+        )
+
 
 class Coord(object):
     """ Coordinates of a syntactic element. Consists of:
@@ -523,6 +597,7 @@ class VPPAPIParser(object):
                 | option
                 | import
                 | enum
+                | enumflag
                 | union
                 | service
                 | paths
@@ -643,6 +718,17 @@ class VPPAPIParser(object):
         else:
             p[0] = Enum(p[2], p[4])
 
+    def p_enumflag(self, p):
+        '''enumflag : ENUMFLAG ID '{' enum_statements '}' ';' '''
+        p[0] = EnumFlag(p[2], p[4])
+
+    def p_enumflag_type(self, p):
+        ''' enumflag : ENUMFLAG ID ':' enum_size '{' enum_statements '}' ';' '''  # noqa : E502
+        if len(p) == 9:
+            p[0] = EnumFlag(p[2], p[6], enumtype=p[4])
+        else:
+            p[0] = EnumFlag(p[2], p[4])
+
     def p_enum_size(self, p):
         ''' enum_size : U8
                       | U16
@@ -902,14 +988,6 @@ class VPPAPI():
                 print('File not found: {}'.format(filename), file=sys.stderr)
                 sys.exit(2)
 
-    def autoreply_block(self, name, parent):
-        block = [Field('u32', 'context'),
-                 Field('i32', 'retval')]
-        # inherhit the parent's options
-        for k, v in parent.options.items():
-            block.append(Option(k, v))
-        return Define(name + '_reply', [], block)
-
     def process(self, objs):
         s = {}
         s['Option'] = {}
@@ -921,32 +999,17 @@ class VPPAPI():
         s['Paths'] = []
         crc = 0
         for o in objs:
-            tname = o.__class__.__name__
             try:
                 crc = binascii.crc32(o.crc, crc) & 0xffffffff
             except AttributeError:
                 pass
-            if isinstance(o, Define):
-                s[tname].append(o)
-                if o.autoreply:
-                    s[tname].append(self.autoreply_block(o.name, o))
-            elif isinstance(o, Option):
-                s[tname][o.option] = o.value
-            elif type(o) is list:
+
+            if type(o) is list:
                 for o2 in o:
                     if isinstance(o2, Service):
-                        s['Service'].append(o2)
-            elif isinstance(o, (Enum, Typedef, Union, Using)):
-                s['types'].append(o)
-            elif isinstance(o, Counter):
-                s['Counters'].append(o)
-            elif isinstance(o, Paths):
-                s['Paths'].append(o)
+                        o2.process(s)
             else:
-                if tname not in s:
-                    raise ValueError('Unknown class type: {} {}'
-                                     .format(tname, o))
-                s[tname].append(o)
+                o.process(s)
 
         msgs = {d.name: d for d in s['Define']}
         svcs = {s.caller: s for s in s['Service']}
@@ -1021,7 +1084,7 @@ class VPPAPI():
 
         return s
 
-    def process_imports(self, objs, in_import, result):
+    def process_imports(self, objs, in_import, result):  # -> List
         for o in objs:
             # Only allow the following object types from imported file
             if in_import and not isinstance(o, (Enum, Import, Typedef,
diff --git a/src/tools/vppapigen/vppapigen_c.py b/src/tools/vppapigen/vppapigen_c.py
index 4369dd86690..44f86be9222 100644
--- a/src/tools/vppapigen/vppapigen_c.py
+++ b/src/tools/vppapigen/vppapigen_c.py
@@ -32,6 +32,7 @@ import shutil
 
 process_imports = False
 
+
 ###############################################################################
 class ToJSON():
     '''Class to generate functions converting from VPP binary API to JSON.'''
@@ -89,7 +90,7 @@ class ToJSON():
             return 'cJSON_AddBoolToObject', '', False
 
         # Lookup type name check if it's enum
-        if vt.type == 'Enum':
+        if vt.type == 'Enum' or vt.type == 'EnumFlag':
             return '{t}_tojson'.format(t=t), '', True
         return '{t}_tojson'.format(t=t), '&', True
 
@@ -186,6 +187,7 @@ class ToJSON():
         write('}\n')
 
     _dispatch['Enum'] = print_enum
+    _dispatch['EnumFlag'] = print_enum
 
     def print_typedef(self, o):
         '''Create cJSON (dictionary) object from VPP API typedef'''
@@ -454,6 +456,7 @@ class FromJSON():
         write('}\n')
 
     _dispatch['Enum'] = print_enum
+    _dispatch['EnumFlag'] = print_enum
 
     def print_typedef(self, o):
         '''Convert from JSON object to VPP API binary representation'''
@@ -845,6 +848,7 @@ class Printfun():
         write('    }\n')
 
     _dispatch['Enum'] = print_enum
+    _dispatch['EnumFlag'] = print_enum
 
     def print_obj(self, o, stream):
         '''Entry point'''
@@ -935,7 +939,7 @@ static inline u8 *format_vl_api_{name}_t (u8 *s, va_list * args)
 '''
 
     for t in objs:
-        if t.__class__.__name__ == 'Enum':
+        if t.__class__.__name__ == 'Enum' or t.__class__.__name__ == 'EnumFlag':
             write(signature.format(name=t.name))
             pp.print_enum(t.block, stream)
             write('    return s;\n')
@@ -1071,7 +1075,7 @@ static inline void vl_api_{name}_t_endian (vl_api_{name}_t *a)
 '''
 
     for t in objs:
-        if t.__class__.__name__ == 'Enum':
+        if t.__class__.__name__ == 'Enum' or t.__class__.__name__ == 'EnumFlag' :
             output += signature.format(name=t.name)
             if t.enumtype in ENDIAN_STRINGS:
                 output += ('    *a = {}(*a);\n'
@@ -1191,7 +1195,7 @@ def generate_include_types(s, module, stream):
                       (o.alias['type'], o.name, o.alias['length']))
             else:
                 write('typedef %s vl_api_%s_t;\n' % (o.alias['type'], o.name))
-        elif tname == 'Enum':
+        elif tname == 'Enum' or tname == 'EnumFlag':
             if o.enumtype == 'u32':
                 write("typedef enum {\n")
             else:
diff --git a/src/tools/vppapigen/vppapigen_json.py b/src/tools/vppapigen/vppapigen_json.py
index 19f7d65b8c5..93ec21fcc88 100644
--- a/src/tools/vppapigen/vppapigen_json.py
+++ b/src/tools/vppapigen/vppapigen_json.py
@@ -96,6 +96,8 @@ def run(args, filename, s):
                               if o.__class__.__name__ == 'Union']))
     j['enums'] = (walk_enums([o for o in s['types']
                               if o.__class__.__name__ == 'Enum']))
+    j['enumflags'] = (walk_enums([o for o in s['types']
+                                  if o.__class__.__name__ == 'EnumFlag']))
     j['services'] = walk_services(s['Service'])
     j['options'] = s['Option']
     j['aliases'] = {o.name:o.alias for o in s['types'] if o.__class__.__name__ == 'Using'}