From 9d42087149a6870965896be74dc6260f72d2cac9 Mon Sep 17 00:00:00 2001 From: Ole Troan Date: Thu, 12 Oct 2017 13:06:35 +0200 Subject: VPPAPIGEN: vppapigen replacement in Python PLY. This is a version of the VPP API generator in Python PLY. It supports the existing language, and has a plugin architecture for generators. Currently C and JSON are supported. Changes: - vl_api_version to option version = "major.minor.patch" - enum support - Added error checking and reporting - import support (removed the C pre-processor) - services (tying request/reply together) Version: option version = "1.0.0"; Enum: enum colours { RED, BLUE = 50, }; define foo { vl_api_colours_t colours; }; Services: service { rpc foo returns foo_reply; rpc foo_dump returns stream foo_details; rpc want_stats returns want_stats_reply events ip4_counters, ip6_counters; }; Future planned features: - unions - bool, text - array support (including length) - proto3 output plugin - Refactor C/C++ generator as a plugin - Refactor Java generator as a plugin Change-Id: Ifa289966c790e1b1a8e2938a91e69331e3a58bdf Signed-off-by: Ole Troan --- src/tools/vppapigen/vppapigen.py | 745 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 745 insertions(+) create mode 100755 src/tools/vppapigen/vppapigen.py (limited to 'src/tools/vppapigen/vppapigen.py') diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py new file mode 100755 index 00000000000..81f26fe4432 --- /dev/null +++ b/src/tools/vppapigen/vppapigen.py @@ -0,0 +1,745 @@ +#!/usr/bin/env python + +from __future__ import print_function +import ply.lex as lex +import ply.yacc as yacc +import sys +import argparse +import logging +import binascii +import os + +# +# VPP API language +# + +# Global dictionary of new types (including enums) +global_types = {} + + +def global_type_add(name): + '''Add new type to the dictionary of types ''' + type_name = 'vl_api_' + name + '_t' + if type_name in global_types: + raise KeyError('Type is already defined: {}'.format(name)) + global_types[type_name] = True + + +# All your trace are belong to us! +def exception_handler(exception_type, exception, traceback): + print ("%s: %s" % (exception_type.__name__, exception)) + + +# +# Lexer +# +class VPPAPILexer(object): + def __init__(self, filename): + self.filename = filename + + reserved = { + 'service': 'SERVICE', + 'rpc': 'RPC', + 'returns': 'RETURNS', + 'stream': 'STREAM', + 'events': 'EVENTS', + 'define': 'DEFINE', + 'typedef': 'TYPEDEF', + 'enum': 'ENUM', + 'typeonly': 'TYPEONLY', + 'manual_print': 'MANUAL_PRINT', + 'manual_endian': 'MANUAL_ENDIAN', + 'dont_trace': 'DONT_TRACE', + 'autoreply': 'AUTOREPLY', + 'option': 'OPTION', + 'u8': 'U8', + 'u16': 'U16', + 'u32': 'U32', + 'u64': 'U64', + 'i8': 'I8', + 'i16': 'I16', + 'i32': 'I32', + 'i64': 'I64', + 'f64': 'F64', + 'bool': 'BOOL', + 'string': 'STRING', + 'import': 'IMPORT', + 'true': 'TRUE', + 'false': 'FALSE', + } + + tokens = ['STRING_LITERAL', + 'ID', 'NUM'] + list(reserved.values()) + + t_ignore_LINE_COMMENT = '//.*' + + def t_NUM(self, t): + r'0[xX][0-9a-fA-F]+|\d+' + base = 16 if t.value.startswith('0x') else 10 + t.value = int(t.value, base) + return t + + def t_ID(self, t): + r'[a-zA-Z_][a-zA-Z_0-9]*' + # Check for reserved words + t.type = VPPAPILexer.reserved.get(t.value, 'ID') + return t + + # C string + def t_STRING_LITERAL(self, t): + r'\"([^\\\n]|(\\.))*?\"' + t.value = str(t.value).replace("\"", "") + return t + + # C or C++ comment (ignore) + def t_comment(self, t): + r'(/\*(.|\n)*?\*/)|(//.*)' + t.lexer.lineno += t.value.count('\n') + + # Error handling rule + def t_error(self, t): + raise ParseError("Illegal character '{}' ({})" + "in {}: line {}".format(t.value[0], + hex(ord(t.value[0])), + self.filename, + t.lexer.lineno)) + t.lexer.skip(1) + + # Define a rule so we can track line numbers + def t_newline(self, t): + r'\n+' + t.lexer.lineno += len(t.value) + + literals = ":{}[];=.," + + # A string containing ignored characters (spaces and tabs) + t_ignore = ' \t' + + +class Iterator(type): + def __iter__(self): + return self.iter() + + +class Service(): + def __init__(self, caller, reply, events=[], stream=False): + self.caller = caller + self.reply = reply + self.stream = stream + self.events = events + + +class Typedef(): + def __init__(self, name, flags, block): + self.name = name + self.flags = flags + self.block = block + self.crc = binascii.crc32(str(block)) & 0xffffffff + global_type_add(name) + + def __repr__(self): + return self.name + str(self.flags) + str(self.block) + + +class Define(): + def __init__(self, name, flags, block): + self.name = name + self.flags = flags + self.block = block + self.crc = binascii.crc32(str(block)) & 0xffffffff + self.typeonly = False + self.dont_trace = False + self.manual_print = False + self.manual_endian = False + self.autoreply = False + self.singular = False + for f in flags: + if f == 'typeonly': + self.typeonly = True + global_type_add(name) + elif f == 'dont_trace': + self.dont_trace = True + elif f == 'manual_print': + self.manual_print = True + elif f == 'manual_endian': + self.manual_endian = True + elif f == 'autoreply': + self.autoreply = True + + for b in block: + if isinstance(b, Option): + if b[1] == 'singular' and b[2] == 'true': + self.singular = True + block.remove(b) + + def __repr__(self): + return self.name + str(self.flags) + str(self.block) + + +class Enum(): + def __init__(self, name, block, enumtype='u32'): + self.name = name + self.enumtype = enumtype + count = 0 + for i, b in enumerate(block): + if type(b) is list: + count = b[1] + else: + count += 1 + block[i] = [b, count] + + self.block = block + self.crc = binascii.crc32(str(block)) & 0xffffffff + global_type_add(name) + + def __repr__(self): + return self.name + str(self.block) + + +class Import(): + def __init__(self, filename): + self.filename = filename + + # Deal with imports + parser = VPPAPI(filename=filename) + dirlist = dirlist_get() + f = filename + for dir in dirlist: + f = os.path.join(dir, filename) + if os.path.exists(f): + break + with open(f) as fd: + self.result = parser.parse_file(fd, None) + + def __repr__(self): + return self.filename + + +class Option(): + def __init__(self, option): + self.option = option + self.crc = binascii.crc32(str(option)) & 0xffffffff + + def __repr__(self): + return str(self.option) + + def __getitem__(self, index): + return self.option[index] + + +class Array(): + def __init__(self, fieldtype, name, length): + self.type = 'Array' + self.fieldtype = fieldtype + self.fieldname = name + if type(length) is str: + self.lengthfield = length + self.length = 0 + else: + self.length = length + self.lengthfield = None + + def __repr__(self): + return str([self.fieldtype, self.fieldname, self.length, + self.lengthfield]) + + +class Field(): + def __init__(self, fieldtype, name): + self.type = 'Field' + self.fieldtype = fieldtype + self.fieldname = name + + def __repr__(self): + return str([self.fieldtype, self.fieldname]) + + +class Coord(object): + """ Coordinates of a syntactic element. Consists of: + - File name + - Line number + - (optional) column number, for the Lexer + """ + __slots__ = ('file', 'line', 'column', '__weakref__') + + def __init__(self, file, line, column=None): + self.file = file + self.line = line + self.column = column + + def __str__(self): + str = "%s:%s" % (self.file, self.line) + if self.column: + str += ":%s" % self.column + return str + + +class ParseError(Exception): + pass + + +# +# Grammar rules +# +class VPPAPIParser(object): + tokens = VPPAPILexer.tokens + + def __init__(self, filename, logger): + self.filename = filename + self.logger = logger + self.fields = [] + + def _parse_error(self, msg, coord): + raise ParseError("%s: %s" % (coord, msg)) + + def _parse_warning(self, msg, coord): + if self.logger: + self.logger.warning("%s: %s" % (coord, msg)) + + def _coord(self, lineno, column=None): + return Coord( + file=self.filename, + line=lineno, column=column) + + def _token_coord(self, p, token_idx): + """ Returns the coordinates for the YaccProduction object 'p' indexed + with 'token_idx'. The coordinate includes the 'lineno' and + 'column'. Both follow the lex semantic, starting from 1. + """ + last_cr = p.lexer.lexdata.rfind('\n', 0, p.lexpos(token_idx)) + if last_cr < 0: + last_cr = -1 + column = (p.lexpos(token_idx) - (last_cr)) + return self._coord(p.lineno(token_idx), column) + + def p_slist(self, p): + '''slist : stmt + | slist stmt''' + if len(p) == 2: + p[0] = [p[1]] + else: + p[0] = p[1] + [p[2]] + + def p_stmt(self, p): + '''stmt : define + | typedef + | option + | import + | enum + | service''' + p[0] = p[1] + + def p_import(self, p): + '''import : IMPORT STRING_LITERAL ';' ''' + p[0] = Import(p[2]) + + def p_service(self, p): + '''service : SERVICE '{' service_statements '}' ';' ''' + p[0] = p[3] + + def p_service_statements(self, p): + '''service_statements : service_statement + | service_statements service_statement''' + if len(p) == 2: + p[0] = [p[1]] + else: + p[0] = p[1] + [p[2]] + + def p_service_statement(self, p): + '''service_statement : RPC ID RETURNS ID ';' + | RPC ID RETURNS STREAM ID ';' + | RPC ID RETURNS ID EVENTS event_list ';' ''' + if len(p) == 8: + p[0] = Service(p[2], p[4], p[6]) + elif len(p) == 7: + p[0] = Service(p[2], p[5], stream=True) + else: + p[0] = Service(p[2], p[4]) + + def p_event_list(self, p): + '''event_list : events + | event_list events ''' + if len(p) == 2: + p[0] = [p[1]] + else: + p[0] = p[1] + [p[2]] + + def p_event(self, p): + '''events : ID + | ID ',' ''' + p[0] = p[1] + + def p_enum(self, p): + '''enum : ENUM ID '{' enum_statements '}' ';' ''' + p[0] = Enum(p[2], p[4]) + + def p_enum_type(self, p): + ''' enum : ENUM ID ':' enum_size '{' enum_statements '}' ';' ''' + if len(p) == 9: + p[0] = Enum(p[2], p[6], enumtype=p[4]) + else: + p[0] = Enum(p[2], p[4]) + + def p_enum_size(self, p): + ''' enum_size : U8 + | U16 + | U32 ''' + p[0] = p[1] + + def p_define(self, p): + '''define : DEFINE ID '{' block_statements_opt '}' ';' ''' + self.fields = [] + p[0] = Define(p[2], [], p[4]) + + def p_define_flist(self, p): + '''define : flist DEFINE ID '{' block_statements_opt '}' ';' ''' + p[0] = Define(p[3], p[1], p[5]) + + def p_flist(self, p): + '''flist : flag + | flist flag''' + if len(p) == 2: + p[0] = [p[1]] + else: + p[0] = p[1] + [p[2]] + + def p_flag(self, p): + '''flag : MANUAL_PRINT + | MANUAL_ENDIAN + | DONT_TRACE + | TYPEONLY + | AUTOREPLY''' + if len(p) == 1: + return + p[0] = p[1] + + def p_typedef(self, p): + '''typedef : TYPEDEF ID '{' block_statements_opt '}' ';' ''' + p[0] = Typedef(p[2], [], p[4]) + + def p_block_statements_opt(self, p): + '''block_statements_opt : block_statements''' + p[0] = p[1] + + def p_block_statements(self, p): + '''block_statements : block_statement + | block_statements block_statement''' + if len(p) == 2: + p[0] = [p[1]] + else: + p[0] = p[1] + [p[2]] + + def p_block_statement(self, p): + '''block_statement : declaration + | option ''' + p[0] = p[1] + + def p_enum_statements(self, p): + '''enum_statements : enum_statement + | enum_statements enum_statement''' + if len(p) == 2: + p[0] = [p[1]] + else: + p[0] = p[1] + [p[2]] + + def p_enum_statement(self, p): + '''enum_statement : ID '=' NUM ',' + | ID ',' ''' + if len(p) == 5: + p[0] = [p[1], p[3]] + else: + p[0] = p[1] + + def p_declaration(self, p): + '''declaration : type_specifier ID ';' ''' + if len(p) != 4: + self._parse_error('ERROR') + self.fields.append(p[2]) + p[0] = Field(p[1], p[2]) + + def p_declaration_array(self, p): + '''declaration : type_specifier ID '[' NUM ']' ';' + | type_specifier ID '[' ID ']' ';' ''' + if len(p) != 7: + return self._parse_error( + 'array: %s' % p.value, + self._coord(lineno=p.lineno)) + + # Make this error later + if type(p[4]) is int and p[4] == 0: + # XXX: Line number is wrong + self._parse_warning('Old Style VLA: {} {}[{}];' + .format(p[1], p[2], p[4]), + self._token_coord(p, 1)) + + if type(p[4]) is str and p[4] not in self.fields: + # Verify that length field exists + self._parse_error('Missing length field: {} {}[{}];' + .format(p[1], p[2], p[4]), + self._token_coord(p, 1)) + p[0] = Array(p[1], p[2], p[4]) + + def p_option(self, p): + '''option : OPTION ID '=' assignee ';' ''' + p[0] = Option([p[1], p[2], p[4]]) + + def p_assignee(self, p): + '''assignee : NUM + | TRUE + | FALSE + | STRING_LITERAL ''' + p[0] = p[1] + + def p_type_specifier(self, p): + '''type_specifier : U8 + | U16 + | U32 + | U64 + | I8 + | I16 + | I32 + | I64 + | F64 + | BOOL + | STRING''' + p[0] = p[1] + + # Do a second pass later to verify that user defined types are defined + def p_typedef_specifier(self, p): + '''type_specifier : ID ''' + if p[1] not in global_types: + self._parse_error('Undefined type: {}'.format(p[1]), + self._token_coord(p, 1)) + p[0] = p[1] + + # Error rule for syntax errors + def p_error(self, p): + if p: + self._parse_error( + 'before: %s' % p.value, + self._coord(lineno=p.lineno)) + else: + self._parse_error('At end of input', self.filename) + + +class VPPAPI(object): + + def __init__(self, debug=False, filename='', logger=None): + self.lexer = lex.lex(module=VPPAPILexer(filename), debug=debug) + self.parser = yacc.yacc(module=VPPAPIParser(filename, logger), + tabmodule='vppapigentab', debug=debug) + self.logger = logger + + def parse_string(self, code, debug=0, lineno=1): + self.lexer.lineno = lineno + return self.parser.parse(code, lexer=self.lexer, debug=debug) + + def parse_file(self, fd, debug=0): + data = fd.read() + return self.parse_string(data, debug=debug) + + def autoreply_block(self, name): + block = [Field('u32', 'context'), + Field('i32', 'retval')] + return Define(name + '_reply', [], block) + + def process(self, objs): + s = {} + s['defines'] = [] + s['typedefs'] = [] + s['imports'] = [] + s['options'] = {} + s['enums'] = [] + s['services'] = [] + + for o in objs: + if isinstance(o, Define): + if o.typeonly: + s['typedefs'].append(o) + else: + s['defines'].append(o) + if o.autoreply: + s['defines'].append(self.autoreply_block(o.name)) + elif isinstance(o, Option): + s['options'][o[1]] = o[2] + elif isinstance(o, Enum): + s['enums'].append(o) + elif isinstance(o, Typedef): + s['typedefs'].append(o) + elif type(o) is list: + for o2 in o: + if isinstance(o2, Service): + s['services'].append(o2) + + # Create services implicitly + msgs = {d.name: d for d in s['defines']} + svcs = {s.caller: s for s in s['services']} + + for service in svcs: + if service not in msgs: + raise ValueError('Service definition refers to unknown message' + ' definition: {}'.format(service)) + if svcs[service].reply not in msgs: + raise ValueError('Service definition refers to unknown message' + ' definition in reply: {}' + .format(svcs[service].reply)) + for event in svcs[service].events: + if event not in msgs: + raise ValueError('Service definition refers to unknown ' + 'event: {} in message: {}' + .format(event, service)) + + for d in msgs: + if msgs[d].singular is True: + continue + if d.endswith('_counters'): + continue + if d.endswith('_reply'): + if d[:-6] in svcs: + continue + if d[:-6] not in msgs: + self.logger.warning('{} missing calling message' + .format(d)) + continue + if d.endswith('_dump'): + if d in svcs: + continue + if d[:-5]+'_details' in msgs: + s['services'].append(Service(d, d[:-5]+'_details', + stream=True)) + else: + self.logger.error('{} missing details message' + .format(d)) + continue + + if d.endswith('_details'): + if d[:-8]+'_dump' not in msgs: + self.logger.error('{} missing dump message' + .format(d)) + continue + + if d in svcs: + continue + if d+'_reply' in msgs: + s['services'].append(Service(d, d+'_reply')) + else: + self.logger.warning('{} missing reply message ({})' + .format(d, d+'_reply')) + s['services'].append(Service(d, None)) + + return s + + def process_imports(self, objs): + for o in objs: + if isinstance(o, Import): + return objs + self.process_imports(o.result) + return objs + + +# Add message ids to each message. +def add_msg_id(s): + for o in s: + o.block.insert(0, Field('u16', '_vl_msg_id')) + return s + + +def getcrc(s): + return binascii.crc32(str(s)) & 0xffffffff + + +dirlist = [] + + +def dirlist_add(dirs): + global dirlist + if dirs: + dirlist = dirlist + dirs + + +def dirlist_get(): + return dirlist + + +# +# Main +# +def main(): + logging.basicConfig() + log = logging.getLogger('vppapigen') + + cliparser = argparse.ArgumentParser(description='VPP API generator') + cliparser.add_argument('--pluginpath', default=""), + cliparser.add_argument('--includedir', action='append'), + cliparser.add_argument('--input', type=argparse.FileType('r'), + default=sys.stdin) + cliparser.add_argument('--output', nargs='?', type=argparse.FileType('w'), + default=sys.stdout) + + cliparser.add_argument('output_module', nargs='?', default='C') + cliparser.add_argument('--debug', action='store_true') + cliparser.add_argument('--show-name', nargs=1) + args = cliparser.parse_args() + + dirlist_add(args.includedir) + if not args.debug: + sys.excepthook = exception_handler + + # Filename + if args.show_name: + filename = args.show_name[0] + elif args.input != sys.stdin: + filename = args.input.name + else: + filename = '' + + parser = VPPAPI(debug=args.debug, filename=filename, logger=log) + result = parser.parse_file(args.input, log) + + # Build a list of objects. Hash of lists. + result = parser.process_imports(result) + s = parser.process(result) + + # Add msg_id field + s['defines'] = add_msg_id(s['defines']) + + file_crc = getcrc(s) + + # + # Debug + if args.debug: + import pprint + pp = pprint.PrettyPrinter(indent=4) + for t in s['defines']: + pp.pprint([t.name, t.flags, t.block]) + for t in s['typedefs']: + pp.pprint([t.name, t.flags, t.block]) + + # + # Generate representation + # + import imp + + # Default path + if not args.pluginpath: + pluginpath = os.path.dirname(os.path.realpath(__file__)) + \ + '/../share/vpp/' + else: + pluginpath = args.pluginpath + '/' + module_path = pluginpath + args.output_module + '.py' + + try: + plugin = imp.load_source(args.output_module, module_path) + except Exception, err: + raise Exception('Error importing output plugin: {}, {}' + .format(module_path, err)) + + result = plugin.run(filename, s, file_crc) + if result: + print (result, file=args.output) + else: + raise Exception('Running plugin failed: {} {}' + .format(filename, result)) + + +if __name__ == '__main__': + main() -- cgit 1.2.3-korg