vppapigen: api crc checker

crcchecker is a tool for enforcement of the binary API.

1. Production APIs should never change.
2. An API can be deprecated across three release cycles.
   Release 1: API is marked as deprecated.
   option deprecated="vyy.mm";
   option replaced_by="new_api_2";
   Release 2: both old and new APIs are supported
   Release 3: the deprecated API is deleted.
3. APIs that are experimental / not released are not checked.
   An API message can be individually marked as in progress, by:
   option status="in_progress";
   In the API definition.

The definition of a "production API" is if the major version in the API file is > 0.

extras/scripts/crcchecker.py --check-patchset # returns -1 if backwards incompatible
extras/scripts/crcchecker.py --dump-manifest
extras/scripts/crcchecker.py --git-revision v20.01 <files>
extras/scripts/crcchecker.py -- diff <oldfile> <newfile>

This patch integrates the tool in "make checkstyle-api".
A future patch is required to integrate the tool in the verify job.
I.e. this patch does not enable enforcement through Jenkins.

Change-Id: I5f13c0976d8a12a58131b3e270f2dc9c00dc7d8c
Type: feature
Signed-off-by: Ole Troan <ot@cisco.com>

2 files changed, 67 insertions, 23 deletions

diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py
index b17ad6d15c9..28712e4d949 100755
--- a/src/tools/vppapigen/vppapigen.py
+++ b/src/tools/vppapigen/vppapigen.py
@@ -9,6 +9,7 @@ import logging
 import binascii
 import os
 import sys
+from subprocess import Popen, PIPE
 
 log = logging.getLogger('vppapigen')
 
@@ -275,14 +276,17 @@ class Define():
             elif f == 'autoreply':
                 self.autoreply = True
 
+        remove = []
         for b in block:
             if isinstance(b, Option):
                 if b[1] == 'singular' and b[2] == 'true':
                     self.singular = True
                 else:
                     self.options[b.option] = b.value
-                block.remove(b)
+                remove.append(b)
 
+        block = [x for x in block if not x in remove]
+        self.block = block
         self.vla = vla_is_last_check(name, block)
         self.crc = str(block).encode()
 
@@ -322,25 +326,20 @@ class Import():
 
         return seen_imports[args[0]]
 
-    def __init__(self, filename):
+    def __init__(self, filename, revision):
         if self._initialized:
             return
         else:
             self.filename = filename
             # Deal with imports
-            parser = VPPAPI(filename=filename)
+            parser = VPPAPI(filename=filename, revision=revision)
             dirlist = dirlist_get()
             f = filename
             for dir in dirlist:
                 f = os.path.join(dir, filename)
                 if os.path.exists(f):
                     break
-            if sys.version[0] == '2':
-                with open(f) as fd:
-                    self.result = parser.parse_file(fd, None)
-            else:
-                with open(f, encoding='utf-8') as fd:
-                    self.result = parser.parse_file(fd, None)
+            self.result = parser.parse_filename(f, None)
             self._initialized = True
 
     def __repr__(self):
@@ -430,10 +429,11 @@ class ParseError(Exception):
 class VPPAPIParser(object):
     tokens = VPPAPILexer.tokens
 
-    def __init__(self, filename, logger):
+    def __init__(self, filename, logger, revision=None):
         self.filename = filename
         self.logger = logger
         self.fields = []
+        self.revision = revision
 
     def _parse_error(self, msg, coord):
         raise ParseError("%s: %s" % (coord, msg))
@@ -478,7 +478,7 @@ class VPPAPIParser(object):
 
     def p_import(self, p):
         '''import : IMPORT STRING_LITERAL ';' '''
-        p[0] = Import(p[2])
+        p[0] = Import(p[2], revision=self.revision)
 
     def p_service(self, p):
         '''service : SERVICE '{' service_statements '}' ';' '''
@@ -731,23 +731,42 @@ class VPPAPIParser(object):
 
 class VPPAPI(object):
 
-    def __init__(self, debug=False, filename='', logger=None):
+    def __init__(self, debug=False, filename='', logger=None, revision=None):
         self.lexer = lex.lex(module=VPPAPILexer(filename), debug=debug)
-        self.parser = yacc.yacc(module=VPPAPIParser(filename, logger),
+        self.parser = yacc.yacc(module=VPPAPIParser(filename, logger,
+                                                    revision=revision),
                                 write_tables=False, debug=debug)
         self.logger = logger
+        self.revision = revision
+        self.filename = filename
 
     def parse_string(self, code, debug=0, lineno=1):
         self.lexer.lineno = lineno
         return self.parser.parse(code, lexer=self.lexer, debug=debug)
 
-    def parse_file(self, fd, debug=0):
+    def parse_fd(self, fd, debug=0):
         data = fd.read()
         return self.parse_string(data, debug=debug)
 
-    def autoreply_block(self, name):
+    def parse_filename(self, filename, debug=0):
+        if self.revision:
+            git_show = f'git show  {self.revision}:{filename}'
+            with Popen(git_show.split(), stdout=PIPE, encoding='utf-8') as git:
+                return self.parse_fd(git.stdout, None)
+        else:
+            try:
+                with open(filename, encoding='utf-8') as fd:
+                    return self.parse_fd(fd, None)
+            except FileNotFoundError:
+                print(f'File not found: {filename}', file=sys.stderr)
+                sys.exit(2)
+
+    def autoreply_block(self, name, parent):
         block = [Field('u32', 'context'),
                  Field('i32', 'retval')]
+        # inherhit the parent's options
+        for k,v in parent.options.items():
+            block.append(Option(k, v))
         return Define(name + '_reply', [], block)
 
     def process(self, objs):
@@ -767,7 +786,7 @@ class VPPAPI(object):
             if isinstance(o, Define):
                 s[tname].append(o)
                 if o.autoreply:
-                    s[tname].append(self.autoreply_block(o.name))
+                    s[tname].append(self.autoreply_block(o.name, o))
             elif isinstance(o, Option):
                 s[tname][o.option] = o.value
             elif type(o) is list:
@@ -925,9 +944,7 @@ def main():
     cliparser.add_argument('--pluginpath', default=""),
     cliparser.add_argument('--includedir', action='append'),
     cliparser.add_argument('--outputdir', action='store'),
-    cliparser.add_argument('--input',
-                           type=argparse.FileType('r', encoding='UTF-8'),
-                           default=sys.stdin)
+    cliparser.add_argument('--input')
     cliparser.add_argument('--output', nargs='?',
                            type=argparse.FileType('w', encoding='UTF-8'),
                            default=sys.stdout)
@@ -935,6 +952,8 @@ def main():
     cliparser.add_argument('output_module', nargs='?', default='C')
     cliparser.add_argument('--debug', action='store_true')
     cliparser.add_argument('--show-name', nargs=1)
+    cliparser.add_argument('--git-revision',
+                           help="Git revision to use for opening files")
     args = cliparser.parse_args()
 
     dirlist_add(args.includedir)
@@ -944,8 +963,8 @@ def main():
     # Filename
     if args.show_name:
         filename = args.show_name[0]
-    elif args.input != sys.stdin:
-        filename = args.input.name
+    elif args.input:
+        filename = args.input
     else:
         filename = ''
 
@@ -954,8 +973,17 @@ def main():
     else:
         logging.basicConfig()
 
-    parser = VPPAPI(debug=args.debug, filename=filename, logger=log)
-    parsed_objects = parser.parse_file(args.input, log)
+    parser = VPPAPI(debug=args.debug, filename=filename, logger=log,
+                    revision=args.git_revision)
+
+    try:
+        if not args.input:
+            parsed_objects = parser.parse_fd(sys.stdin, log)
+        else:
+            parsed_objects = parser.parse_filename(args.input, log)
+    except ParseError as e:
+        print('Parse error: ', e, file=sys.stderr)
+        sys.exit(1)
 
     # Build a list of objects. Hash of lists.
     result = []
diff --git a/src/tools/vppapigen/vppapigen_crc.py b/src/tools/vppapigen/vppapigen_crc.py
new file mode 100644
index 00000000000..b3cb5855c9d
--- /dev/null
+++ b/src/tools/vppapigen/vppapigen_crc.py
@@ -0,0 +1,16 @@
+# CRC generation
+import json
+
+#
+# Plugin entry point
+#
+def run(args, input_filename, s):
+    j = {}
+    major = 0
+    if 'version' in s['Option']:
+        v = s['Option']['version']
+        (major, minor, patch) = v.split('.')
+    for t in s['Define']:
+        j[t.name] = {'crc': f'{t.crc:#08x}', 'version': major,
+                     'options': t.options}
+    return json.dumps(j, indent=4, separators=(',', ': '))