summaryrefslogtreecommitdiffstats
path: root/extras/scripts
diff options
context:
space:
mode:
authorKlement Sekera <klement.sekera@gmail.com>2022-04-26 19:02:15 +0200
committerOle Tr�an <otroan@employees.org>2022-05-10 18:52:08 +0000
commitd9b0c6fbf7aa5bd9af84264105b39c82028a4a29 (patch)
tree4f786cfd8ebc2443cb11e11b74c8657204068898 /extras/scripts
parentf90348bcb4afd0af2611cefc43b17ef3042b511c (diff)
tests: replace pycodestyle with black
Drop pycodestyle for code style checking in favor of black. Black is much faster, stable PEP8 compliant code style checker offering also automatic formatting. It aims to be very stable and produce smallest diffs. It's used by many small and big projects. Running checkstyle with black takes a few seconds with a terse output. Thus, test-checkstyle-diff is no longer necessary. Expand scope of checkstyle to all python files in the repo, replacing test-checkstyle with checkstyle-python. Also, fixstyle-python is now available for automatic style formatting. Note: python virtualenv has been consolidated in test/Makefile, test/requirements*.txt which will eventually be moved to a central location. This is required to simply the automated generation of docker executor images in the CI. Type: improvement Change-Id: I022a326603485f58585e879ac0f697fceefbc9c8 Signed-off-by: Klement Sekera <klement.sekera@gmail.com> Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
Diffstat (limited to 'extras/scripts')
-rwxr-xr-xextras/scripts/crcchecker.py207
-rwxr-xr-xextras/scripts/list_api_changes.py23
2 files changed, 125 insertions, 105 deletions
diff --git a/extras/scripts/crcchecker.py b/extras/scripts/crcchecker.py
index f3021c3c8b6..01cb02523d0 100755
--- a/extras/scripts/crcchecker.py
+++ b/extras/scripts/crcchecker.py
@@ -1,9 +1,9 @@
#!/usr/bin/env python3
-'''
+"""
crcchecker is a tool to used to enforce that .api messages do not change.
API files with a semantic version < 1.0.0 are ignored.
-'''
+"""
import sys
import os
@@ -14,67 +14,75 @@ from subprocess import run, PIPE, check_output, CalledProcessError
# pylint: disable=subprocess-run-check
-ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + '/../..'
-APIGENBIN = f'{ROOTDIR}/src/tools/vppapigen/vppapigen.py'
+ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + "/../.."
+APIGENBIN = f"{ROOTDIR}/src/tools/vppapigen/vppapigen.py"
def crc_from_apigen(revision, filename):
- '''Runs vppapigen with crc plugin returning a JSON object with CRCs for
- all APIs in filename'''
+ """Runs vppapigen with crc plugin returning a JSON object with CRCs for
+ all APIs in filename"""
if not revision and not os.path.isfile(filename):
- print(f'skipping: {filename}', file=sys.stderr)
+ print(f"skipping: {filename}", file=sys.stderr)
# Return <class 'set'> instead of <class 'dict'>
return {-1}
if revision:
- apigen = (f'{APIGENBIN} --git-revision {revision} --includedir src '
- f'--input {filename} CRC')
+ apigen = (
+ f"{APIGENBIN} --git-revision {revision} --includedir src "
+ f"--input {filename} CRC"
+ )
else:
- apigen = (f'{APIGENBIN} --includedir src --input {filename} CRC')
+ apigen = f"{APIGENBIN} --includedir src --input {filename} CRC"
returncode = run(apigen.split(), stdout=PIPE, stderr=PIPE)
if returncode.returncode == 2: # No such file
- print(f'skipping: {revision}:{filename} {returncode}', file=sys.stderr)
+ print(f"skipping: {revision}:{filename} {returncode}", file=sys.stderr)
return {}
if returncode.returncode != 0:
- print(f'vppapigen failed for {revision}:{filename} with '
- 'command\n {apigen}\n error: {rv}',
- returncode.stderr.decode('ascii'), file=sys.stderr)
+ print(
+ f"vppapigen failed for {revision}:{filename} with "
+ "command\n {apigen}\n error: {rv}",
+ returncode.stderr.decode("ascii"),
+ file=sys.stderr,
+ )
sys.exit(-2)
return json.loads(returncode.stdout)
def dict_compare(dict1, dict2):
- '''Compare two dictionaries returning added, removed, modified
- and equal entries'''
+ """Compare two dictionaries returning added, removed, modified
+ and equal entries"""
d1_keys = set(dict1.keys())
d2_keys = set(dict2.keys())
intersect_keys = d1_keys.intersection(d2_keys)
added = d1_keys - d2_keys
removed = d2_keys - d1_keys
- modified = {o: (dict1[o], dict2[o]) for o in intersect_keys
- if dict1[o]['crc'] != dict2[o]['crc']}
+ modified = {
+ o: (dict1[o], dict2[o])
+ for o in intersect_keys
+ if dict1[o]["crc"] != dict2[o]["crc"]
+ }
same = set(o for o in intersect_keys if dict1[o] == dict2[o])
return added, removed, modified, same
def filelist_from_git_ls():
- '''Returns a list of all api files in the git repository'''
+ """Returns a list of all api files in the git repository"""
filelist = []
- git_ls = 'git ls-files *.api'
+ git_ls = "git ls-files *.api"
returncode = run(git_ls.split(), stdout=PIPE, stderr=PIPE)
if returncode.returncode != 0:
sys.exit(returncode.returncode)
- for line in returncode.stdout.decode('ascii').split('\n'):
+ for line in returncode.stdout.decode("ascii").split("\n"):
if line:
filelist.append(line)
return filelist
def is_uncommitted_changes():
- '''Returns true if there are uncommitted changes in the repo'''
- git_status = 'git status --porcelain -uno'
+ """Returns true if there are uncommitted changes in the repo"""
+ git_status = "git status --porcelain -uno"
returncode = run(git_status.split(), stdout=PIPE, stderr=PIPE)
if returncode.returncode != 0:
sys.exit(returncode.returncode)
@@ -85,27 +93,29 @@ def is_uncommitted_changes():
def filelist_from_git_grep(filename):
- '''Returns a list of api files that this <filename> api files imports.'''
+ """Returns a list of api files that this <filename> api files imports."""
filelist = []
try:
- returncode = check_output(f'git grep -e "import .*{filename}"'
- ' -- *.api',
- shell=True)
+ returncode = check_output(
+ f'git grep -e "import .*{filename}"' " -- *.api", shell=True
+ )
except CalledProcessError:
return []
- for line in returncode.decode('ascii').split('\n'):
+ for line in returncode.decode("ascii").split("\n"):
if line:
- filename, _ = line.split(':')
+ filename, _ = line.split(":")
filelist.append(filename)
return filelist
def filelist_from_patchset(pattern):
- '''Returns list of api files in changeset and the list of api
- files they import.'''
+ """Returns list of api files in changeset and the list of api
+ files they import."""
filelist = []
- git_cmd = ('((git diff HEAD~1.. --name-only;git ls-files -m) | '
- 'sort -u | grep "\\.api$")')
+ git_cmd = (
+ "((git diff HEAD~1.. --name-only;git ls-files -m) | "
+ 'sort -u | grep "\\.api$")'
+ )
try:
res = check_output(git_cmd, shell=True)
except CalledProcessError:
@@ -113,7 +123,7 @@ def filelist_from_patchset(pattern):
# Check for dependencies (imports)
imported_files = []
- for line in res.decode('ascii').split('\n'):
+ for line in res.decode("ascii").split("\n"):
if not line:
continue
if not re.search(pattern, line):
@@ -126,88 +136,91 @@ def filelist_from_patchset(pattern):
def is_deprecated(message):
- '''Given a message, return True if message is deprecated'''
- if 'options' in message:
- if 'deprecated' in message['options']:
+ """Given a message, return True if message is deprecated"""
+ if "options" in message:
+ if "deprecated" in message["options"]:
return True
# recognize the deprecated format
- if 'status' in message['options'] and \
- message['options']['status'] == 'deprecated':
+ if (
+ "status" in message["options"]
+ and message["options"]["status"] == "deprecated"
+ ):
print("WARNING: please use 'option deprecated;'")
return True
return False
def is_in_progress(message):
- '''Given a message, return True if message is marked as in_progress'''
- if 'options' in message:
- if 'in_progress' in message['options']:
+ """Given a message, return True if message is marked as in_progress"""
+ if "options" in message:
+ if "in_progress" in message["options"]:
return True
# recognize the deprecated format
- if 'status' in message['options'] and \
- message['options']['status'] == 'in_progress':
+ if (
+ "status" in message["options"]
+ and message["options"]["status"] == "in_progress"
+ ):
print("WARNING: please use 'option in_progress;'")
return True
return False
def report(new, old):
- '''Given a dictionary of new crcs and old crcs, print all the
+ """Given a dictionary of new crcs and old crcs, print all the
added, removed, modified, in-progress, deprecated messages.
- Return the number of backwards incompatible changes made.'''
+ Return the number of backwards incompatible changes made."""
# pylint: disable=too-many-branches
- new.pop('_version', None)
- old.pop('_version', None)
+ new.pop("_version", None)
+ old.pop("_version", None)
added, removed, modified, _ = dict_compare(new, old)
backwards_incompatible = 0
# print the full list of in-progress messages
# they should eventually either disappear of become supported
for k in new.keys():
- newversion = int(new[k]['version'])
+ newversion = int(new[k]["version"])
if newversion == 0 or is_in_progress(new[k]):
- print(f'in-progress: {k}')
+ print(f"in-progress: {k}")
for k in added:
- print(f'added: {k}')
+ print(f"added: {k}")
for k in removed:
- oldversion = int(old[k]['version'])
- if oldversion > 0 and not is_deprecated(old[k]) and not \
- is_in_progress(old[k]):
+ oldversion = int(old[k]["version"])
+ if oldversion > 0 and not is_deprecated(old[k]) and not is_in_progress(old[k]):
backwards_incompatible += 1
- print(f'removed: ** {k}')
+ print(f"removed: ** {k}")
else:
- print(f'removed: {k}')
+ print(f"removed: {k}")
for k in modified.keys():
- oldversion = int(old[k]['version'])
- newversion = int(new[k]['version'])
+ oldversion = int(old[k]["version"])
+ newversion = int(new[k]["version"])
if oldversion > 0 and not is_in_progress(old[k]):
backwards_incompatible += 1
- print(f'modified: ** {k}')
+ print(f"modified: ** {k}")
else:
- print(f'modified: {k}')
+ print(f"modified: {k}")
# check which messages are still there but were marked for deprecation
for k in new.keys():
- newversion = int(new[k]['version'])
+ newversion = int(new[k]["version"])
if newversion > 0 and is_deprecated(new[k]):
if k in old:
if not is_deprecated(old[k]):
- print(f'deprecated: {k}')
+ print(f"deprecated: {k}")
else:
- print(f'added+deprecated: {k}')
+ print(f"added+deprecated: {k}")
return backwards_incompatible
def check_patchset():
- '''Compare the changes to API messages in this changeset.
+ """Compare the changes to API messages in this changeset.
Ignores API files with version < 1.0.0.
Only considers API files located under the src directory in the repo.
- '''
- files = filelist_from_patchset('^src/')
- revision = 'HEAD~1'
+ """
+ files = filelist_from_patchset("^src/")
+ revision = "HEAD~1"
oldcrcs = {}
newcrcs = {}
@@ -216,7 +229,7 @@ def check_patchset():
_ = crc_from_apigen(None, filename)
# Ignore removed files
if isinstance(_, set) == 0:
- if isinstance(_, set) == 0 and _['_version']['major'] == '0':
+ if isinstance(_, set) == 0 and _["_version"]["major"] == "0":
continue
newcrcs.update(_)
@@ -225,27 +238,31 @@ def check_patchset():
backwards_incompatible = report(newcrcs, oldcrcs)
if backwards_incompatible:
# alert on changing production API
- print("crcchecker: Changing production APIs in an incompatible way",
- file=sys.stderr)
+ print(
+ "crcchecker: Changing production APIs in an incompatible way",
+ file=sys.stderr,
+ )
sys.exit(-1)
else:
- print('*' * 67)
- print('* VPP CHECKAPI SUCCESSFULLY COMPLETED')
- print('*' * 67)
+ print("*" * 67)
+ print("* VPP CHECKAPI SUCCESSFULLY COMPLETED")
+ print("*" * 67)
def main():
- '''Main entry point.'''
- parser = argparse.ArgumentParser(description='VPP CRC checker.')
- parser.add_argument('--git-revision',
- help='Git revision to compare against')
- parser.add_argument('--dump-manifest', action='store_true',
- help='Dump CRC for all messages')
- parser.add_argument('--check-patchset', action='store_true',
- help='Check patchset for backwards incompatbile changes')
- parser.add_argument('files', nargs='*')
- parser.add_argument('--diff', help='Files to compare (on filesystem)',
- nargs=2)
+ """Main entry point."""
+ parser = argparse.ArgumentParser(description="VPP CRC checker.")
+ parser.add_argument("--git-revision", help="Git revision to compare against")
+ parser.add_argument(
+ "--dump-manifest", action="store_true", help="Dump CRC for all messages"
+ )
+ parser.add_argument(
+ "--check-patchset",
+ action="store_true",
+ help="Check patchset for backwards incompatbile changes",
+ )
+ parser.add_argument("files", nargs="*")
+ parser.add_argument("--diff", help="Files to compare (on filesystem)", nargs=2)
args = parser.parse_args()
@@ -267,17 +284,16 @@ def main():
for filename in files:
crcs.update(crc_from_apigen(args.git_revision, filename))
for k, value in crcs.items():
- print(f'{k}: {value}')
+ print(f"{k}: {value}")
sys.exit(0)
# Find changes between current patchset and given revision (previous)
if args.check_patchset:
if args.git_revision:
- print('Argument git-revision ignored', file=sys.stderr)
+ print("Argument git-revision ignored", file=sys.stderr)
# Check there are no uncomitted changes
if is_uncommitted_changes():
- print('Please stash or commit changes in workspace',
- file=sys.stderr)
+ print("Please stash or commit changes in workspace", file=sys.stderr)
sys.exit(-1)
check_patchset()
sys.exit(0)
@@ -286,7 +302,7 @@ def main():
# Find changes between a given file and a revision
files = args.files if args.files else filelist_from_git_ls()
- revision = args.git_revision if args.git_revision else 'HEAD~1'
+ revision = args.git_revision if args.git_revision else "HEAD~1"
oldcrcs = {}
newcrcs = {}
@@ -299,13 +315,16 @@ def main():
if args.check_patchset:
if backwards_incompatible:
# alert on changing production API
- print("crcchecker: Changing production APIs in an incompatible way", file=sys.stderr)
+ print(
+ "crcchecker: Changing production APIs in an incompatible way",
+ file=sys.stderr,
+ )
sys.exit(-1)
else:
- print('*' * 67)
- print('* VPP CHECKAPI SUCCESSFULLY COMPLETED')
- print('*' * 67)
+ print("*" * 67)
+ print("* VPP CHECKAPI SUCCESSFULLY COMPLETED")
+ print("*" * 67)
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/extras/scripts/list_api_changes.py b/extras/scripts/list_api_changes.py
index ac8cf477810..7433f542aaa 100755
--- a/extras/scripts/list_api_changes.py
+++ b/extras/scripts/list_api_changes.py
@@ -4,21 +4,21 @@ import fnmatch
import os
import subprocess
-starttag = 'v19.08-rc0'
-endtag = 'HEAD'
+starttag = "v19.08-rc0"
+endtag = "HEAD"
emit_md = True
apifiles = []
-for root, dirnames, filenames in os.walk('.'):
- for filename in fnmatch.filter(filenames, '*.api'):
+for root, dirnames, filenames in os.walk("."):
+ for filename in fnmatch.filter(filenames, "*.api"):
apifiles.append(os.path.join(root, filename))
for f in apifiles:
- commits = subprocess.check_output(['git', 'log',
- '--oneline', starttag + '..' + endtag,
- f])
+ commits = subprocess.check_output(
+ ["git", "log", "--oneline", starttag + ".." + endtag, f]
+ )
if commits:
- if f[0:2] == './':
+ if f[0:2] == "./":
f = f[2:]
if emit_md:
print("| @c %s ||" % f)
@@ -27,9 +27,10 @@ for f in apifiles:
parts = line.strip().split()
commit = parts[0]
message = b" ".join(parts[1:]).decode().replace("|", r"\|")
- print("| [%s](https://gerrit.fd.io/r/gitweb?"
- "p=vpp.git;a=commit;h=%s) | %s |" % (
- commit, commit, message))
+ print(
+ "| [%s](https://gerrit.fd.io/r/gitweb?"
+ "p=vpp.git;a=commit;h=%s) | %s |" % (commit, commit, message)
+ )
print()
else:
print(f)