diff options
Diffstat (limited to 'extras/scripts/crcchecker.py')
| -rwxr-xr-x | extras/scripts/crcchecker.py | 218 |
1 files changed, 148 insertions, 70 deletions
diff --git a/extras/scripts/crcchecker.py b/extras/scripts/crcchecker.py index 2b026338129..5060d9f2e0e 100755 --- a/extras/scripts/crcchecker.py +++ b/extras/scripts/crcchecker.py @@ -1,134 +1,176 @@ #!/usr/bin/env python3 +''' +crcchecker is a tool to used to enforce that .api messages do not change. +API files with a semantic version < 1.0.0 are ignored. +''' + import sys import os import json import argparse +import re from subprocess import run, PIPE, check_output, CalledProcessError -rootdir = os.path.dirname(os.path.realpath(__file__)) + '/../..' +# pylint: disable=subprocess-run-check + +ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + '/../..' +APIGENBIN = f'{ROOTDIR}/src/tools/vppapigen/vppapigen.py' + def crc_from_apigen(revision, filename): + '''Runs vppapigen with crc plugin returning a JSON object with CRCs for + all APIs in filename''' if not revision and not os.path.isfile(filename): print(f'skipping: {filename}', file=sys.stderr) return {} - apigen_bin = f'{rootdir}/src/tools/vppapigen/vppapigen.py' + if revision: - apigen = (f'{apigen_bin} --git-revision {revision} --includedir src ' + apigen = (f'{APIGENBIN} --git-revision {revision} --includedir src ' f'--input {filename} CRC') else: - apigen = (f'{apigen_bin} --includedir src --input {filename} CRC') - rv = run(apigen.split(), stdout=PIPE, stderr=PIPE) - if rv.returncode == 2: # No such file - print(f'skipping: {revision}:{filename} {rv}', file=sys.stderr) + apigen = (f'{APIGENBIN} --includedir src --input {filename} CRC') + returncode = run(apigen.split(), stdout=PIPE, stderr=PIPE) + if returncode.returncode == 2: # No such file + print(f'skipping: {revision}:{filename} {returncode}', file=sys.stderr) return {} - if rv.returncode != 0: - print(f'vppapigen failed for {revision}:{filename} with command\n {apigen}\n error: {rv}', - rv.stderr.decode('ascii'), file=sys.stderr) + if returncode.returncode != 0: + print(f'vppapigen failed for {revision}:{filename} with ' + 'command\n {apigen}\n error: {rv}', + returncode.stderr.decode('ascii'), file=sys.stderr) sys.exit(-2) - return json.loads(rv.stdout) + return json.loads(returncode.stdout) -def dict_compare(d1, d2): - d1_keys = set(d1.keys()) - d2_keys = set(d2.keys()) +def dict_compare(dict1, dict2): + '''Compare two dictionaries returning added, removed, modified + and equal entries''' + d1_keys = set(dict1.keys()) + d2_keys = set(dict2.keys()) intersect_keys = d1_keys.intersection(d2_keys) added = d1_keys - d2_keys removed = d2_keys - d1_keys - modified = {o: (d1[o], d2[o]) for o in intersect_keys if d1[o]['crc'] != d2[o]['crc']} - same = set(o for o in intersect_keys if d1[o] == d2[o]) + modified = {o: (dict1[o], dict2[o]) for o in intersect_keys + if dict1[o]['crc'] != dict2[o]['crc']} + same = set(o for o in intersect_keys if dict1[o] == dict2[o]) return added, removed, modified, same def filelist_from_git_ls(): + '''Returns a list of all api files in the git repository''' filelist = [] git_ls = 'git ls-files *.api' - rv = run(git_ls.split(), stdout=PIPE, stderr=PIPE) - if rv.returncode != 0: - sys.exit(rv.returncode) + returncode = run(git_ls.split(), stdout=PIPE, stderr=PIPE) + if returncode.returncode != 0: + sys.exit(returncode.returncode) - for l in rv.stdout.decode('ascii').split('\n'): - if len(l): - filelist.append(l) + for line in returncode.stdout.decode('ascii').split('\n'): + if line: + filelist.append(line) return filelist def is_uncommitted_changes(): + '''Returns true if there are uncommitted changes in the repo''' git_status = 'git status --porcelain -uno' - rv = run(git_status.split(), stdout=PIPE, stderr=PIPE) - if rv.returncode != 0: - sys.exit(rv.returncode) + returncode = run(git_status.split(), stdout=PIPE, stderr=PIPE) + if returncode.returncode != 0: + sys.exit(returncode.returncode) - if len(rv.stdout): + if returncode.stdout: return True return False def filelist_from_git_grep(filename): + '''Returns a list of api files that this <filename> api files imports.''' filelist = [] try: - rv = check_output(f'git grep -e "import .*{filename}" -- *.api', shell=True) - except CalledProcessError as err: + returncode = check_output(f'git grep -e "import .*{filename}"' + ' -- *.api', + shell=True) + except CalledProcessError: return [] - print('RV', err.returncode) - for l in rv.decode('ascii').split('\n'): - if l: - f, p = l.split(':') - filelist.append(f) + for line in returncode.decode('ascii').split('\n'): + if line: + filename, _ = line.split(':') + filelist.append(filename) return filelist -def filelist_from_patchset(): +def filelist_from_patchset(pattern): + '''Returns list of api files in changeset and the list of api + files they import.''' filelist = [] - git_cmd = '((git diff HEAD~1.. --name-only;git ls-files -m) | sort -u)' - rv = check_output(git_cmd, shell=True) - for l in rv.decode('ascii').split('\n'): - if len(l) and os.path.splitext(l)[1] == '.api': - filelist.append(l) + git_cmd = ('((git diff HEAD~1.. --name-only;git ls-files -m) | ' + 'sort -u | grep "\\.api$")') + returncode = check_output(git_cmd, shell=True) # Check for dependencies (imports) imported_files = [] - for f in filelist: - imported_files.extend(filelist_from_git_grep(os.path.basename(f))) + for line in returncode.decode('ascii').split('\n'): + if not line: + continue + if not re.search(pattern, line): + continue + filelist.append(line) + imported_files.extend(filelist_from_git_grep(os.path.basename(line))) filelist.extend(imported_files) return set(filelist) -def is_deprecated(d, k): - if 'options' in d[k]: - if 'deprecated' in d[k]['options']: + +def is_deprecated(message): + '''Given a message, return True if message is deprecated''' + if 'options' in message: + if 'deprecated' in message['options']: return True # recognize the deprecated format - if 'status' in d[k]['options'] and d[k]['options']['status'] == 'deprecated': + if 'status' in message['options'] and \ + message['options']['status'] == 'deprecated': print("WARNING: please use 'option deprecated;'") return True return False -def is_in_progress(d, k): - if 'options' in d[k]: - if 'in_progress' in d[k]['options']: + +def is_in_progress(message): + '''Given a message, return True if message is marked as in_progress''' + if 'options' in message: + if 'in_progress' in message['options']: return True # recognize the deprecated format - if 'status' in d[k]['options'] and d[k]['options']['status'] == 'in_progress': + if 'status' in message['options'] and \ + message['options']['status'] == 'in_progress': print("WARNING: please use 'option in_progress;'") return True return False + def report(new, old): - added, removed, modified, same = dict_compare(new, old) + '''Given a dictionary of new crcs and old crcs, print all the + added, removed, modified, in-progress, deprecated messages. + Return the number of backwards incompatible changes made.''' + + # pylint: disable=too-many-branches + + new.pop('_version', None) + old.pop('_version', None) + added, removed, modified, _ = dict_compare(new, old) backwards_incompatible = 0 + # print the full list of in-progress messages # they should eventually either disappear of become supported for k in new.keys(): newversion = int(new[k]['version']) - if newversion == 0 or is_in_progress(new, k): + if newversion == 0 or is_in_progress(new[k]): print(f'in-progress: {k}') for k in added: print(f'added: {k}') for k in removed: oldversion = int(old[k]['version']) - if oldversion > 0 and not is_deprecated(old, k) and not is_in_progress(old, k): + if oldversion > 0 and not is_deprecated(old[k]) and not \ + is_in_progress(old[k]): backwards_incompatible += 1 print(f'removed: ** {k}') else: @@ -136,7 +178,7 @@ def report(new, old): for k in modified.keys(): oldversion = int(old[k]['version']) newversion = int(new[k]['version']) - if oldversion > 0 and not is_in_progress(old, k): + if oldversion > 0 and not is_in_progress(old[k]): backwards_incompatible += 1 print(f'modified: ** {k}') else: @@ -145,9 +187,9 @@ def report(new, old): # check which messages are still there but were marked for deprecation for k in new.keys(): newversion = int(new[k]['version']) - if newversion > 0 and is_deprecated(new, k): + if newversion > 0 and is_deprecated(new[k]): if k in old: - if not is_deprecated(old, k): + if not is_deprecated(old[k]): print(f'deprecated: {k}') else: print(f'added+deprecated: {k}') @@ -155,16 +197,49 @@ def report(new, old): return backwards_incompatible +def check_patchset(): + '''Compare the changes to API messages in this changeset. + Ignores API files with version < 1.0.0. + Only considers API files located under the src directory in the repo. + ''' + files = filelist_from_patchset('^src/') + revision = 'HEAD~1' + + oldcrcs = {} + newcrcs = {} + for filename in files: + # Ignore files that have version < 1.0.0 + _ = crc_from_apigen(None, filename) + if _['_version']['major'] == '0': + continue + + newcrcs.update(_) + oldcrcs.update(crc_from_apigen(revision, filename)) + + backwards_incompatible = report(newcrcs, oldcrcs) + if backwards_incompatible: + # alert on changing production API + print("crcchecker: Changing production APIs in an incompatible way", + file=sys.stderr) + sys.exit(-1) + else: + print('*' * 67) + print('* VPP CHECKAPI SUCCESSFULLY COMPLETED') + print('*' * 67) + + def main(): + '''Main entry point.''' parser = argparse.ArgumentParser(description='VPP CRC checker.') parser.add_argument('--git-revision', help='Git revision to compare against') parser.add_argument('--dump-manifest', action='store_true', help='Dump CRC for all messages') parser.add_argument('--check-patchset', action='store_true', - help='Dump CRC for all messages') + help='Check patchset for backwards incompatbile changes') parser.add_argument('files', nargs='*') - parser.add_argument('--diff', help='Files to compare (on filesystem)', nargs=2) + parser.add_argument('--diff', help='Files to compare (on filesystem)', + nargs=2) args = parser.parse_args() @@ -183,10 +258,10 @@ def main(): if args.dump_manifest: files = args.files if args.files else filelist_from_git_ls() crcs = {} - for f in files: - crcs.update(crc_from_apigen(args.git_revision, f)) - for k, v in crcs.items(): - print(f'{k}: {v}') + for filename in files: + crcs.update(crc_from_apigen(args.git_revision, filename)) + for k, value in crcs.items(): + print(f'{k}: {value}') sys.exit(0) # Find changes between current patchset and given revision (previous) @@ -195,21 +270,23 @@ def main(): print('Argument git-revision ignored', file=sys.stderr) # Check there are no uncomitted changes if is_uncommitted_changes(): - print('Please stash or commit changes in workspace', file=sys.stderr) + print('Please stash or commit changes in workspace', + file=sys.stderr) sys.exit(-1) - files = filelist_from_patchset() - else: - # Find changes between current workspace and revision - # Find changes between a given file and a revision - files = args.files if args.files else filelist_from_git_ls() + check_patchset() + sys.exit(0) + + # Find changes between current workspace and revision + # Find changes between a given file and a revision + files = args.files if args.files else filelist_from_git_ls() revision = args.git_revision if args.git_revision else 'HEAD~1' oldcrcs = {} newcrcs = {} - for f in files: - newcrcs.update(crc_from_apigen(None, f)) - oldcrcs.update(crc_from_apigen(revision, f)) + for file in files: + newcrcs.update(crc_from_apigen(None, file)) + oldcrcs.update(crc_from_apigen(revision, file)) backwards_incompatible = report(newcrcs, oldcrcs) @@ -223,5 +300,6 @@ def main(): print('* VPP CHECKAPI SUCCESSFULLY COMPLETED') print('*' * 67) + if __name__ == '__main__': main() |