tests: replace pycodestyle with black

Drop pycodestyle for code style checking in favor of black. Black is
much faster, stable PEP8 compliant code style checker offering also
automatic formatting. It aims to be very stable and produce smallest
diffs. It's used by many small and big projects.

Running checkstyle with black takes a few seconds with a terse output.
Thus, test-checkstyle-diff is no longer necessary.

Expand scope of checkstyle to all python files in the repo, replacing
test-checkstyle with checkstyle-python.

Also, fixstyle-python is now available for automatic style formatting.

Note: python virtualenv has been consolidated in test/Makefile,
test/requirements*.txt which will eventually be moved to a central
location.  This is required to simply the automated generation of
docker executor images in the CI.

Type: improvement
Change-Id: I022a326603485f58585e879ac0f697fceefbc9c8
Signed-off-by: Klement Sekera <klement.sekera@gmail.com>
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

2 files changed, 125 insertions, 105 deletions

diff --git a/extras/scripts/crcchecker.py b/extras/scripts/crcchecker.py
index f3021c3c8b6..01cb02523d0 100755
--- a/extras/scripts/crcchecker.py
+++ b/extras/scripts/crcchecker.py
@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 
-'''
+"""
 crcchecker is a tool to used to enforce that .api messages do not change.
 API files with a semantic version < 1.0.0 are ignored.
-'''
+"""
 
 import sys
 import os
@@ -14,67 +14,75 @@ from subprocess import run, PIPE, check_output, CalledProcessError
 
 # pylint: disable=subprocess-run-check
 
-ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + '/../..'
-APIGENBIN = f'{ROOTDIR}/src/tools/vppapigen/vppapigen.py'
+ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + "/../.."
+APIGENBIN = f"{ROOTDIR}/src/tools/vppapigen/vppapigen.py"
 
 
 def crc_from_apigen(revision, filename):
-    '''Runs vppapigen with crc plugin returning a JSON object with CRCs for
-    all APIs in filename'''
+    """Runs vppapigen with crc plugin returning a JSON object with CRCs for
+    all APIs in filename"""
     if not revision and not os.path.isfile(filename):
-        print(f'skipping: {filename}', file=sys.stderr)
+        print(f"skipping: {filename}", file=sys.stderr)
         # Return <class 'set'> instead of <class 'dict'>
         return {-1}
 
     if revision:
-        apigen = (f'{APIGENBIN} --git-revision {revision} --includedir src '
-                  f'--input {filename} CRC')
+        apigen = (
+            f"{APIGENBIN} --git-revision {revision} --includedir src "
+            f"--input {filename} CRC"
+        )
     else:
-        apigen = (f'{APIGENBIN} --includedir src --input {filename} CRC')
+        apigen = f"{APIGENBIN} --includedir src --input {filename} CRC"
     returncode = run(apigen.split(), stdout=PIPE, stderr=PIPE)
     if returncode.returncode == 2:  # No such file
-        print(f'skipping: {revision}:{filename} {returncode}', file=sys.stderr)
+        print(f"skipping: {revision}:{filename} {returncode}", file=sys.stderr)
         return {}
     if returncode.returncode != 0:
-        print(f'vppapigen failed for {revision}:{filename} with '
-              'command\n {apigen}\n error: {rv}',
-              returncode.stderr.decode('ascii'), file=sys.stderr)
+        print(
+            f"vppapigen failed for {revision}:{filename} with "
+            "command\n {apigen}\n error: {rv}",
+            returncode.stderr.decode("ascii"),
+            file=sys.stderr,
+        )
         sys.exit(-2)
 
     return json.loads(returncode.stdout)
 
 
 def dict_compare(dict1, dict2):
-    '''Compare two dictionaries returning added, removed, modified
-    and equal entries'''
+    """Compare two dictionaries returning added, removed, modified
+    and equal entries"""
     d1_keys = set(dict1.keys())
     d2_keys = set(dict2.keys())
     intersect_keys = d1_keys.intersection(d2_keys)
     added = d1_keys - d2_keys
     removed = d2_keys - d1_keys
-    modified = {o: (dict1[o], dict2[o]) for o in intersect_keys
-                if dict1[o]['crc'] != dict2[o]['crc']}
+    modified = {
+        o: (dict1[o], dict2[o])
+        for o in intersect_keys
+        if dict1[o]["crc"] != dict2[o]["crc"]
+    }
     same = set(o for o in intersect_keys if dict1[o] == dict2[o])
     return added, removed, modified, same
 
 
 def filelist_from_git_ls():
-    '''Returns a list of all api files in the git repository'''
+    """Returns a list of all api files in the git repository"""
     filelist = []
-    git_ls = 'git ls-files *.api'
+    git_ls = "git ls-files *.api"
     returncode = run(git_ls.split(), stdout=PIPE, stderr=PIPE)
     if returncode.returncode != 0:
         sys.exit(returncode.returncode)
 
-    for line in returncode.stdout.decode('ascii').split('\n'):
+    for line in returncode.stdout.decode("ascii").split("\n"):
         if line:
             filelist.append(line)
     return filelist
 
 
 def is_uncommitted_changes():
-    '''Returns true if there are uncommitted changes in the repo'''
-    git_status = 'git status --porcelain -uno'
+    """Returns true if there are uncommitted changes in the repo"""
+    git_status = "git status --porcelain -uno"
     returncode = run(git_status.split(), stdout=PIPE, stderr=PIPE)
     if returncode.returncode != 0:
         sys.exit(returncode.returncode)
@@ -85,27 +93,29 @@ def is_uncommitted_changes():
 
 
 def filelist_from_git_grep(filename):
-    '''Returns a list of api files that this <filename> api files imports.'''
+    """Returns a list of api files that this <filename> api files imports."""
     filelist = []
     try:
-        returncode = check_output(f'git grep -e "import .*{filename}"'
-                                  ' -- *.api',
-                                  shell=True)
+        returncode = check_output(
+            f'git grep -e "import .*{filename}"' " -- *.api", shell=True
+        )
     except CalledProcessError:
         return []
-    for line in returncode.decode('ascii').split('\n'):
+    for line in returncode.decode("ascii").split("\n"):
         if line:
-            filename, _ = line.split(':')
+            filename, _ = line.split(":")
             filelist.append(filename)
     return filelist
 
 
 def filelist_from_patchset(pattern):
-    '''Returns list of api files in changeset and the list of api
-    files they import.'''
+    """Returns list of api files in changeset and the list of api
+    files they import."""
     filelist = []
-    git_cmd = ('((git diff HEAD~1.. --name-only;git ls-files -m) | '
-               'sort -u | grep "\\.api$")')
+    git_cmd = (
+        "((git diff HEAD~1.. --name-only;git ls-files -m) | "
+        'sort -u | grep "\\.api$")'
+    )
     try:
         res = check_output(git_cmd, shell=True)
     except CalledProcessError:
@@ -113,7 +123,7 @@ def filelist_from_patchset(pattern):
 
     # Check for dependencies (imports)
     imported_files = []
-    for line in res.decode('ascii').split('\n'):
+    for line in res.decode("ascii").split("\n"):
         if not line:
             continue
         if not re.search(pattern, line):
@@ -126,88 +136,91 @@ def filelist_from_patchset(pattern):
 
 
 def is_deprecated(message):
-    '''Given a message, return True if message is deprecated'''
-    if 'options' in message:
-        if 'deprecated' in message['options']:
+    """Given a message, return True if message is deprecated"""
+    if "options" in message:
+        if "deprecated" in message["options"]:
             return True
         # recognize the deprecated format
-        if 'status' in message['options'] and \
-           message['options']['status'] == 'deprecated':
+        if (
+            "status" in message["options"]
+            and message["options"]["status"] == "deprecated"
+        ):
             print("WARNING: please use 'option deprecated;'")
             return True
     return False
 
 
 def is_in_progress(message):
-    '''Given a message, return True if message is marked as in_progress'''
-    if 'options' in message:
-        if 'in_progress' in message['options']:
+    """Given a message, return True if message is marked as in_progress"""
+    if "options" in message:
+        if "in_progress" in message["options"]:
             return True
         # recognize the deprecated format
-        if 'status' in message['options'] and \
-           message['options']['status'] == 'in_progress':
+        if (
+            "status" in message["options"]
+            and message["options"]["status"] == "in_progress"
+        ):
             print("WARNING: please use 'option in_progress;'")
             return True
     return False
 
 
 def report(new, old):
-    '''Given a dictionary of new crcs and old crcs, print all the
+    """Given a dictionary of new crcs and old crcs, print all the
     added, removed, modified, in-progress, deprecated messages.
-    Return the number of backwards incompatible changes made.'''
+    Return the number of backwards incompatible changes made."""
 
     # pylint: disable=too-many-branches
 
-    new.pop('_version', None)
-    old.pop('_version', None)
+    new.pop("_version", None)
+    old.pop("_version", None)
     added, removed, modified, _ = dict_compare(new, old)
     backwards_incompatible = 0
 
     # print the full list of in-progress messages
     # they should eventually either disappear of become supported
     for k in new.keys():
-        newversion = int(new[k]['version'])
+        newversion = int(new[k]["version"])
         if newversion == 0 or is_in_progress(new[k]):
-            print(f'in-progress: {k}')
+            print(f"in-progress: {k}")
     for k in added:
-        print(f'added: {k}')
+        print(f"added: {k}")
     for k in removed:
-        oldversion = int(old[k]['version'])
-        if oldversion > 0 and not is_deprecated(old[k]) and not \
-           is_in_progress(old[k]):
+        oldversion = int(old[k]["version"])
+        if oldversion > 0 and not is_deprecated(old[k]) and not is_in_progress(old[k]):
             backwards_incompatible += 1
-            print(f'removed: ** {k}')
+            print(f"removed: ** {k}")
         else:
-            print(f'removed: {k}')
+            print(f"removed: {k}")
     for k in modified.keys():
-        oldversion = int(old[k]['version'])
-        newversion = int(new[k]['version'])
+        oldversion = int(old[k]["version"])
+        newversion = int(new[k]["version"])
         if oldversion > 0 and not is_in_progress(old[k]):
             backwards_incompatible += 1
-            print(f'modified: ** {k}')
+            print(f"modified: ** {k}")
         else:
-            print(f'modified: {k}')
+            print(f"modified: {k}")
 
     # check which messages are still there but were marked for deprecation
     for k in new.keys():
-        newversion = int(new[k]['version'])
+        newversion = int(new[k]["version"])
         if newversion > 0 and is_deprecated(new[k]):
             if k in old:
                 if not is_deprecated(old[k]):
-                    print(f'deprecated: {k}')
+                    print(f"deprecated: {k}")
             else:
-                print(f'added+deprecated: {k}')
+                print(f"added+deprecated: {k}")
 
     return backwards_incompatible
 
 
 def check_patchset():
-    '''Compare the changes to API messages in this changeset.
+    """Compare the changes to API messages in this changeset.
     Ignores API files with version < 1.0.0.
     Only considers API files located under the src directory in the repo.
-    '''
-    files = filelist_from_patchset('^src/')
-    revision = 'HEAD~1'
+    """
+    files = filelist_from_patchset("^src/")
+    revision = "HEAD~1"
 
     oldcrcs = {}
     newcrcs = {}
@@ -216,7 +229,7 @@ def check_patchset():
         _ = crc_from_apigen(None, filename)
         # Ignore removed files
         if isinstance(_, set) == 0:
-            if isinstance(_, set) == 0 and _['_version']['major'] == '0':
+            if isinstance(_, set) == 0 and _["_version"]["major"] == "0":
                 continue
             newcrcs.update(_)
 
@@ -225,27 +238,31 @@ def check_patchset():
     backwards_incompatible = report(newcrcs, oldcrcs)
     if backwards_incompatible:
         # alert on changing production API
-        print("crcchecker: Changing production APIs in an incompatible way",
-              file=sys.stderr)
+        print(
+            "crcchecker: Changing production APIs in an incompatible way",
+            file=sys.stderr,
+        )
         sys.exit(-1)
     else:
-        print('*' * 67)
-        print('* VPP CHECKAPI SUCCESSFULLY COMPLETED')
-        print('*' * 67)
+        print("*" * 67)
+        print("* VPP CHECKAPI SUCCESSFULLY COMPLETED")
+        print("*" * 67)
 
 
 def main():
-    '''Main entry point.'''
-    parser = argparse.ArgumentParser(description='VPP CRC checker.')
-    parser.add_argument('--git-revision',
-                        help='Git revision to compare against')
-    parser.add_argument('--dump-manifest', action='store_true',
-                        help='Dump CRC for all messages')
-    parser.add_argument('--check-patchset', action='store_true',
-                        help='Check patchset for backwards incompatbile changes')
-    parser.add_argument('files', nargs='*')
-    parser.add_argument('--diff', help='Files to compare (on filesystem)',
-                        nargs=2)
+    """Main entry point."""
+    parser = argparse.ArgumentParser(description="VPP CRC checker.")
+    parser.add_argument("--git-revision", help="Git revision to compare against")
+    parser.add_argument(
+        "--dump-manifest", action="store_true", help="Dump CRC for all messages"
+    )
+    parser.add_argument(
+        "--check-patchset",
+        action="store_true",
+        help="Check patchset for backwards incompatbile changes",
+    )
+    parser.add_argument("files", nargs="*")
+    parser.add_argument("--diff", help="Files to compare (on filesystem)", nargs=2)
 
     args = parser.parse_args()
 
@@ -267,17 +284,16 @@ def main():
         for filename in files:
             crcs.update(crc_from_apigen(args.git_revision, filename))
         for k, value in crcs.items():
-            print(f'{k}: {value}')
+            print(f"{k}: {value}")
         sys.exit(0)
 
     # Find changes between current patchset and given revision (previous)
     if args.check_patchset:
         if args.git_revision:
-            print('Argument git-revision ignored', file=sys.stderr)
+            print("Argument git-revision ignored", file=sys.stderr)
         # Check there are no uncomitted changes
         if is_uncommitted_changes():
-            print('Please stash or commit changes in workspace',
-                  file=sys.stderr)
+            print("Please stash or commit changes in workspace", file=sys.stderr)
             sys.exit(-1)
         check_patchset()
         sys.exit(0)
@@ -286,7 +302,7 @@ def main():
     # Find changes between a given file and a revision
     files = args.files if args.files else filelist_from_git_ls()
 
-    revision = args.git_revision if args.git_revision else 'HEAD~1'
+    revision = args.git_revision if args.git_revision else "HEAD~1"
 
     oldcrcs = {}
     newcrcs = {}
@@ -299,13 +315,16 @@ def main():
     if args.check_patchset:
         if backwards_incompatible:
             # alert on changing production API
-            print("crcchecker: Changing production APIs in an incompatible way", file=sys.stderr)
+            print(
+                "crcchecker: Changing production APIs in an incompatible way",
+                file=sys.stderr,
+            )
             sys.exit(-1)
         else:
-            print('*' * 67)
-            print('* VPP CHECKAPI SUCCESSFULLY COMPLETED')
-            print('*' * 67)
+            print("*" * 67)
+            print("* VPP CHECKAPI SUCCESSFULLY COMPLETED")
+            print("*" * 67)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/extras/scripts/list_api_changes.py b/extras/scripts/list_api_changes.py
index ac8cf477810..7433f542aaa 100755
--- a/extras/scripts/list_api_changes.py
+++ b/extras/scripts/list_api_changes.py
@@ -4,21 +4,21 @@ import fnmatch
 import os
 import subprocess
 
-starttag = 'v19.08-rc0'
-endtag = 'HEAD'
+starttag = "v19.08-rc0"
+endtag = "HEAD"
 emit_md = True
 apifiles = []
 
-for root, dirnames, filenames in os.walk('.'):
-    for filename in fnmatch.filter(filenames, '*.api'):
+for root, dirnames, filenames in os.walk("."):
+    for filename in fnmatch.filter(filenames, "*.api"):
         apifiles.append(os.path.join(root, filename))
 
 for f in apifiles:
-    commits = subprocess.check_output(['git', 'log',
-                                       '--oneline', starttag + '..' + endtag,
-                                       f])
+    commits = subprocess.check_output(
+        ["git", "log", "--oneline", starttag + ".." + endtag, f]
+    )
     if commits:
-        if f[0:2] == './':
+        if f[0:2] == "./":
             f = f[2:]
         if emit_md:
             print("| @c %s ||" % f)
@@ -27,9 +27,10 @@ for f in apifiles:
                 parts = line.strip().split()
                 commit = parts[0]
                 message = b" ".join(parts[1:]).decode().replace("|", r"\|")
-                print("| [%s](https://gerrit.fd.io/r/gitweb?"
-                      "p=vpp.git;a=commit;h=%s) | %s |" % (
-                            commit, commit, message))
+                print(
+                    "| [%s](https://gerrit.fd.io/r/gitweb?"
+                    "p=vpp.git;a=commit;h=%s) | %s |" % (commit, commit, message)
+                )
             print()
         else:
             print(f)