diff options
Diffstat (limited to 'extras/scripts')
| -rwxr-xr-x | extras/scripts/crcchecker.py | 207 | ||||
| -rwxr-xr-x | extras/scripts/list_api_changes.py | 23 |
2 files changed, 125 insertions, 105 deletions
diff --git a/extras/scripts/crcchecker.py b/extras/scripts/crcchecker.py index f3021c3c8b6..01cb02523d0 100755 --- a/extras/scripts/crcchecker.py +++ b/extras/scripts/crcchecker.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -''' +""" crcchecker is a tool to used to enforce that .api messages do not change. API files with a semantic version < 1.0.0 are ignored. -''' +""" import sys import os @@ -14,67 +14,75 @@ from subprocess import run, PIPE, check_output, CalledProcessError # pylint: disable=subprocess-run-check -ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + '/../..' -APIGENBIN = f'{ROOTDIR}/src/tools/vppapigen/vppapigen.py' +ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + "/../.." +APIGENBIN = f"{ROOTDIR}/src/tools/vppapigen/vppapigen.py" def crc_from_apigen(revision, filename): - '''Runs vppapigen with crc plugin returning a JSON object with CRCs for - all APIs in filename''' + """Runs vppapigen with crc plugin returning a JSON object with CRCs for + all APIs in filename""" if not revision and not os.path.isfile(filename): - print(f'skipping: {filename}', file=sys.stderr) + print(f"skipping: {filename}", file=sys.stderr) # Return <class 'set'> instead of <class 'dict'> return {-1} if revision: - apigen = (f'{APIGENBIN} --git-revision {revision} --includedir src ' - f'--input {filename} CRC') + apigen = ( + f"{APIGENBIN} --git-revision {revision} --includedir src " + f"--input {filename} CRC" + ) else: - apigen = (f'{APIGENBIN} --includedir src --input {filename} CRC') + apigen = f"{APIGENBIN} --includedir src --input {filename} CRC" returncode = run(apigen.split(), stdout=PIPE, stderr=PIPE) if returncode.returncode == 2: # No such file - print(f'skipping: {revision}:{filename} {returncode}', file=sys.stderr) + print(f"skipping: {revision}:{filename} {returncode}", file=sys.stderr) return {} if returncode.returncode != 0: - print(f'vppapigen failed for {revision}:{filename} with ' - 'command\n {apigen}\n error: {rv}', - returncode.stderr.decode('ascii'), file=sys.stderr) + print( + f"vppapigen failed for {revision}:{filename} with " + "command\n {apigen}\n error: {rv}", + returncode.stderr.decode("ascii"), + file=sys.stderr, + ) sys.exit(-2) return json.loads(returncode.stdout) def dict_compare(dict1, dict2): - '''Compare two dictionaries returning added, removed, modified - and equal entries''' + """Compare two dictionaries returning added, removed, modified + and equal entries""" d1_keys = set(dict1.keys()) d2_keys = set(dict2.keys()) intersect_keys = d1_keys.intersection(d2_keys) added = d1_keys - d2_keys removed = d2_keys - d1_keys - modified = {o: (dict1[o], dict2[o]) for o in intersect_keys - if dict1[o]['crc'] != dict2[o]['crc']} + modified = { + o: (dict1[o], dict2[o]) + for o in intersect_keys + if dict1[o]["crc"] != dict2[o]["crc"] + } same = set(o for o in intersect_keys if dict1[o] == dict2[o]) return added, removed, modified, same def filelist_from_git_ls(): - '''Returns a list of all api files in the git repository''' + """Returns a list of all api files in the git repository""" filelist = [] - git_ls = 'git ls-files *.api' + git_ls = "git ls-files *.api" returncode = run(git_ls.split(), stdout=PIPE, stderr=PIPE) if returncode.returncode != 0: sys.exit(returncode.returncode) - for line in returncode.stdout.decode('ascii').split('\n'): + for line in returncode.stdout.decode("ascii").split("\n"): if line: filelist.append(line) return filelist def is_uncommitted_changes(): - '''Returns true if there are uncommitted changes in the repo''' - git_status = 'git status --porcelain -uno' + """Returns true if there are uncommitted changes in the repo""" + git_status = "git status --porcelain -uno" returncode = run(git_status.split(), stdout=PIPE, stderr=PIPE) if returncode.returncode != 0: sys.exit(returncode.returncode) @@ -85,27 +93,29 @@ def is_uncommitted_changes(): def filelist_from_git_grep(filename): - '''Returns a list of api files that this <filename> api files imports.''' + """Returns a list of api files that this <filename> api files imports.""" filelist = [] try: - returncode = check_output(f'git grep -e "import .*{filename}"' - ' -- *.api', - shell=True) + returncode = check_output( + f'git grep -e "import .*{filename}"' " -- *.api", shell=True + ) except CalledProcessError: return [] - for line in returncode.decode('ascii').split('\n'): + for line in returncode.decode("ascii").split("\n"): if line: - filename, _ = line.split(':') + filename, _ = line.split(":") filelist.append(filename) return filelist def filelist_from_patchset(pattern): - '''Returns list of api files in changeset and the list of api - files they import.''' + """Returns list of api files in changeset and the list of api + files they import.""" filelist = [] - git_cmd = ('((git diff HEAD~1.. --name-only;git ls-files -m) | ' - 'sort -u | grep "\\.api$")') + git_cmd = ( + "((git diff HEAD~1.. --name-only;git ls-files -m) | " + 'sort -u | grep "\\.api$")' + ) try: res = check_output(git_cmd, shell=True) except CalledProcessError: @@ -113,7 +123,7 @@ def filelist_from_patchset(pattern): # Check for dependencies (imports) imported_files = [] - for line in res.decode('ascii').split('\n'): + for line in res.decode("ascii").split("\n"): if not line: continue if not re.search(pattern, line): @@ -126,88 +136,91 @@ def filelist_from_patchset(pattern): def is_deprecated(message): - '''Given a message, return True if message is deprecated''' - if 'options' in message: - if 'deprecated' in message['options']: + """Given a message, return True if message is deprecated""" + if "options" in message: + if "deprecated" in message["options"]: return True # recognize the deprecated format - if 'status' in message['options'] and \ - message['options']['status'] == 'deprecated': + if ( + "status" in message["options"] + and message["options"]["status"] == "deprecated" + ): print("WARNING: please use 'option deprecated;'") return True return False def is_in_progress(message): - '''Given a message, return True if message is marked as in_progress''' - if 'options' in message: - if 'in_progress' in message['options']: + """Given a message, return True if message is marked as in_progress""" + if "options" in message: + if "in_progress" in message["options"]: return True # recognize the deprecated format - if 'status' in message['options'] and \ - message['options']['status'] == 'in_progress': + if ( + "status" in message["options"] + and message["options"]["status"] == "in_progress" + ): print("WARNING: please use 'option in_progress;'") return True return False def report(new, old): - '''Given a dictionary of new crcs and old crcs, print all the + """Given a dictionary of new crcs and old crcs, print all the added, removed, modified, in-progress, deprecated messages. - Return the number of backwards incompatible changes made.''' + Return the number of backwards incompatible changes made.""" # pylint: disable=too-many-branches - new.pop('_version', None) - old.pop('_version', None) + new.pop("_version", None) + old.pop("_version", None) added, removed, modified, _ = dict_compare(new, old) backwards_incompatible = 0 # print the full list of in-progress messages # they should eventually either disappear of become supported for k in new.keys(): - newversion = int(new[k]['version']) + newversion = int(new[k]["version"]) if newversion == 0 or is_in_progress(new[k]): - print(f'in-progress: {k}') + print(f"in-progress: {k}") for k in added: - print(f'added: {k}') + print(f"added: {k}") for k in removed: - oldversion = int(old[k]['version']) - if oldversion > 0 and not is_deprecated(old[k]) and not \ - is_in_progress(old[k]): + oldversion = int(old[k]["version"]) + if oldversion > 0 and not is_deprecated(old[k]) and not is_in_progress(old[k]): backwards_incompatible += 1 - print(f'removed: ** {k}') + print(f"removed: ** {k}") else: - print(f'removed: {k}') + print(f"removed: {k}") for k in modified.keys(): - oldversion = int(old[k]['version']) - newversion = int(new[k]['version']) + oldversion = int(old[k]["version"]) + newversion = int(new[k]["version"]) if oldversion > 0 and not is_in_progress(old[k]): backwards_incompatible += 1 - print(f'modified: ** {k}') + print(f"modified: ** {k}") else: - print(f'modified: {k}') + print(f"modified: {k}") # check which messages are still there but were marked for deprecation for k in new.keys(): - newversion = int(new[k]['version']) + newversion = int(new[k]["version"]) if newversion > 0 and is_deprecated(new[k]): if k in old: if not is_deprecated(old[k]): - print(f'deprecated: {k}') + print(f"deprecated: {k}") else: - print(f'added+deprecated: {k}') + print(f"added+deprecated: {k}") return backwards_incompatible def check_patchset(): - '''Compare the changes to API messages in this changeset. + """Compare the changes to API messages in this changeset. Ignores API files with version < 1.0.0. Only considers API files located under the src directory in the repo. - ''' - files = filelist_from_patchset('^src/') - revision = 'HEAD~1' + """ + files = filelist_from_patchset("^src/") + revision = "HEAD~1" oldcrcs = {} newcrcs = {} @@ -216,7 +229,7 @@ def check_patchset(): _ = crc_from_apigen(None, filename) # Ignore removed files if isinstance(_, set) == 0: - if isinstance(_, set) == 0 and _['_version']['major'] == '0': + if isinstance(_, set) == 0 and _["_version"]["major"] == "0": continue newcrcs.update(_) @@ -225,27 +238,31 @@ def check_patchset(): backwards_incompatible = report(newcrcs, oldcrcs) if backwards_incompatible: # alert on changing production API - print("crcchecker: Changing production APIs in an incompatible way", - file=sys.stderr) + print( + "crcchecker: Changing production APIs in an incompatible way", + file=sys.stderr, + ) sys.exit(-1) else: - print('*' * 67) - print('* VPP CHECKAPI SUCCESSFULLY COMPLETED') - print('*' * 67) + print("*" * 67) + print("* VPP CHECKAPI SUCCESSFULLY COMPLETED") + print("*" * 67) def main(): - '''Main entry point.''' - parser = argparse.ArgumentParser(description='VPP CRC checker.') - parser.add_argument('--git-revision', - help='Git revision to compare against') - parser.add_argument('--dump-manifest', action='store_true', - help='Dump CRC for all messages') - parser.add_argument('--check-patchset', action='store_true', - help='Check patchset for backwards incompatbile changes') - parser.add_argument('files', nargs='*') - parser.add_argument('--diff', help='Files to compare (on filesystem)', - nargs=2) + """Main entry point.""" + parser = argparse.ArgumentParser(description="VPP CRC checker.") + parser.add_argument("--git-revision", help="Git revision to compare against") + parser.add_argument( + "--dump-manifest", action="store_true", help="Dump CRC for all messages" + ) + parser.add_argument( + "--check-patchset", + action="store_true", + help="Check patchset for backwards incompatbile changes", + ) + parser.add_argument("files", nargs="*") + parser.add_argument("--diff", help="Files to compare (on filesystem)", nargs=2) args = parser.parse_args() @@ -267,17 +284,16 @@ def main(): for filename in files: crcs.update(crc_from_apigen(args.git_revision, filename)) for k, value in crcs.items(): - print(f'{k}: {value}') + print(f"{k}: {value}") sys.exit(0) # Find changes between current patchset and given revision (previous) if args.check_patchset: if args.git_revision: - print('Argument git-revision ignored', file=sys.stderr) + print("Argument git-revision ignored", file=sys.stderr) # Check there are no uncomitted changes if is_uncommitted_changes(): - print('Please stash or commit changes in workspace', - file=sys.stderr) + print("Please stash or commit changes in workspace", file=sys.stderr) sys.exit(-1) check_patchset() sys.exit(0) @@ -286,7 +302,7 @@ def main(): # Find changes between a given file and a revision files = args.files if args.files else filelist_from_git_ls() - revision = args.git_revision if args.git_revision else 'HEAD~1' + revision = args.git_revision if args.git_revision else "HEAD~1" oldcrcs = {} newcrcs = {} @@ -299,13 +315,16 @@ def main(): if args.check_patchset: if backwards_incompatible: # alert on changing production API - print("crcchecker: Changing production APIs in an incompatible way", file=sys.stderr) + print( + "crcchecker: Changing production APIs in an incompatible way", + file=sys.stderr, + ) sys.exit(-1) else: - print('*' * 67) - print('* VPP CHECKAPI SUCCESSFULLY COMPLETED') - print('*' * 67) + print("*" * 67) + print("* VPP CHECKAPI SUCCESSFULLY COMPLETED") + print("*" * 67) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/extras/scripts/list_api_changes.py b/extras/scripts/list_api_changes.py index ac8cf477810..7433f542aaa 100755 --- a/extras/scripts/list_api_changes.py +++ b/extras/scripts/list_api_changes.py @@ -4,21 +4,21 @@ import fnmatch import os import subprocess -starttag = 'v19.08-rc0' -endtag = 'HEAD' +starttag = "v19.08-rc0" +endtag = "HEAD" emit_md = True apifiles = [] -for root, dirnames, filenames in os.walk('.'): - for filename in fnmatch.filter(filenames, '*.api'): +for root, dirnames, filenames in os.walk("."): + for filename in fnmatch.filter(filenames, "*.api"): apifiles.append(os.path.join(root, filename)) for f in apifiles: - commits = subprocess.check_output(['git', 'log', - '--oneline', starttag + '..' + endtag, - f]) + commits = subprocess.check_output( + ["git", "log", "--oneline", starttag + ".." + endtag, f] + ) if commits: - if f[0:2] == './': + if f[0:2] == "./": f = f[2:] if emit_md: print("| @c %s ||" % f) @@ -27,9 +27,10 @@ for f in apifiles: parts = line.strip().split() commit = parts[0] message = b" ".join(parts[1:]).decode().replace("|", r"\|") - print("| [%s](https://gerrit.fd.io/r/gitweb?" - "p=vpp.git;a=commit;h=%s) | %s |" % ( - commit, commit, message)) + print( + "| [%s](https://gerrit.fd.io/r/gitweb?" + "p=vpp.git;a=commit;h=%s) | %s |" % (commit, commit, message) + ) print() else: print(f) |