vppapigen: make json in parallel

Type: improvement

This patches makes the make json-api-files
run in parallel in the same python runtime.

Default number of workers is 8, and run time
goes from ~20s to ~2s on average.

Change-Id: Id8cff013889db2671f6b6b4af9a019460c656f81
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>

5 files changed, 140 insertions, 100 deletions

diff --git a/src/tools/vppapigen/generate_json.py b/src/tools/vppapigen/generate_json.py
index e8041c5a3eb..610f84f5533 100755
--- a/src/tools/vppapigen/generate_json.py
+++ b/src/tools/vppapigen/generate_json.py
@@ -16,15 +16,15 @@
 import argparse
 import pathlib
 import subprocess
+import vppapigen
+import os
+from multiprocessing import Pool
 
 BASE_DIR = (
     subprocess.check_output("git rev-parse --show-toplevel", shell=True)
     .strip()
     .decode()
 )
-vppapigen_bin = pathlib.Path(
-    "%s/src/tools/vppapigen/vppapigen.py" % BASE_DIR
-).as_posix()
 
 src_dir_depth = 3
 output_path = pathlib.Path(
@@ -55,32 +55,14 @@ def api_files(src_dir):
     return [x for x in api_search_globs(src_dir)]
 
 
-def vppapigen(vppapigen_bin, output_path, src_dir, src_file):
-    try:
-        subprocess.check_output(
-            [
-                vppapigen_bin,
-                "--includedir",
-                src_dir.as_posix(),
-                "--input",
-                src_file.as_posix(),
-                "JSON",
-                "--output",
-                "%s/%s/%s.json"
-                % (
-                    output_path,
-                    output_dir_map[
-                        src_file.as_posix().split("/")[
-                            src_dir_depth + BASE_DIR.count("/") - 1
-                        ]
-                    ],
-                    src_file.name,
-                ),
-            ]
-        )
-    except KeyError:
-        print("src_file: %s" % src_file)
-        raise
+def get_n_parallel(n_parallel):
+    if n_parallel == 0:
+        n_parallel = os.environ.get("MAKE_PARALLEL_JOBS", os.cpu_count())
+        try:
+            n_parallel = int(n_parallel)
+        except ValueError:
+            return os.cpu_count()
+    return n_parallel or os.cpu_count()
 
 
 def main():
@@ -88,11 +70,15 @@ def main():
     cliparser.add_argument("--srcdir", action="store", default="%s/src" % BASE_DIR),
     cliparser.add_argument("--output", action="store", help="directory to store files"),
     cliparser.add_argument(
+        "--parallel", type=int, default=0, help="Number of parallel processes"
+    ),
+    cliparser.add_argument(
         "--debug-target",
         action="store_true",
         default=False,
         help="'True' if -debug target",
     ),
+
     args = cliparser.parse_args()
 
     src_dir = pathlib.Path(args.srcdir)
@@ -109,8 +95,29 @@ def main():
     for f in output_dir.glob("**/*.api.json"):
         f.unlink()
 
-    for f in api_files(src_dir):
-        vppapigen(vppapigen_bin, output_dir, src_dir, f)
+    with Pool(get_n_parallel(args.parallel)) as p:
+        p.map(
+            vppapigen.run_kw_vppapigen,
+            [
+                {
+                    "output": "%s/%s/%s.json"
+                    % (
+                        output_path,
+                        output_dir_map[
+                            f.as_posix().split("/")[
+                                src_dir_depth + BASE_DIR.count("/") - 1
+                            ]
+                        ],
+                        f.name,
+                    ),
+                    "input_file": f.as_posix(),
+                    "includedir": [src_dir.as_posix()],
+                    "output_module": "JSON",
+                }
+                for f in api_files(src_dir)
+            ],
+        )
+
     print("json files written to: %s/." % output_dir)
 
 
diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py
index 944bf5e7135..9abc5362d80 100755
--- a/src/tools/vppapigen/vppapigen.py
+++ b/src/tools/vppapigen/vppapigen.py
@@ -1152,55 +1152,35 @@ def foldup_crcs(s):
         f.crc = foldup_blocks(f.block, binascii.crc32(f.crc) & 0xFFFFFFFF)
 
 
-#
-# Main
-#
-def main():
-    if sys.version_info < (
-        3,
-        5,
-    ):
-        log.exception(
-            "vppapigen requires a supported version of python. "
-            "Please use version 3.5 or greater. "
-            "Using %s",
-            sys.version,
-        )
-        return 1
-
-    cliparser = argparse.ArgumentParser(description="VPP API generator")
-    cliparser.add_argument("--pluginpath", default="")
-    cliparser.add_argument("--includedir", action="append")
-    cliparser.add_argument("--outputdir", action="store")
-    cliparser.add_argument("--input")
-    cliparser.add_argument(
-        "--output",
-        nargs="?",
-        type=argparse.FileType("w", encoding="UTF-8"),
-        default=sys.stdout,
-    )
-
-    cliparser.add_argument("output_module", nargs="?", default="C")
-    cliparser.add_argument("--debug", action="store_true")
-    cliparser.add_argument("--show-name", nargs=1)
-    cliparser.add_argument(
-        "--git-revision", help="Git revision to use for opening files"
-    )
-    args = cliparser.parse_args()
-
-    dirlist_add(args.includedir)
-    if not args.debug:
+def run_vppapigen(
+    input_file=None,
+    output=sys.stdout,
+    includedir=None,
+    debug=False,
+    show_name=None,
+    output_module="C",
+    outputdir=None,
+    pluginpath="",
+    git_revision=None,
+):
+    # reset globals
+    dirlist.clear()
+    global_types.clear()
+    seen_imports.clear()
+
+    dirlist_add(includedir)
+    if not debug:
         sys.excepthook = exception_handler
 
     # Filename
-    if args.show_name:
-        filename = args.show_name[0]
-    elif args.input:
-        filename = args.input
+    if show_name:
+        filename = show_name[0]
+    elif input_file:
+        filename = input_file
     else:
         filename = ""
 
-    if args.debug:
+    if debug:
         logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
     else:
         logging.basicConfig()
@@ -1212,39 +1192,35 @@ def main():
 
     # Default path
     pluginpath = ""
-    if not args.pluginpath:
+    if not pluginpath:
         cand = []
         cand.append(os.path.dirname(os.path.realpath(__file__)))
         cand.append(os.path.dirname(os.path.realpath(__file__)) + "/../share/vpp/")
         for c in cand:
             c += "/"
-            if os.path.isfile(
-                "{}vppapigen_{}.py".format(c, args.output_module.lower())
-            ):
+            if os.path.isfile("{}vppapigen_{}.py".format(c, output_module.lower())):
                 pluginpath = c
                 break
     else:
-        pluginpath = args.pluginpath + "/"
+        pluginpath = pluginpath + "/"
     if pluginpath == "":
         log.exception("Output plugin not found")
         return 1
-    module_path = "{}vppapigen_{}.py".format(pluginpath, args.output_module.lower())
+    module_path = "{}vppapigen_{}.py".format(pluginpath, output_module.lower())
 
     try:
-        plugin = SourceFileLoader(args.output_module, module_path).load_module()
+        plugin = SourceFileLoader(output_module, module_path).load_module()
     except Exception as err:
         log.exception("Error importing output plugin: %s, %s", module_path, err)
         return 1
 
-    parser = VPPAPI(
-        debug=args.debug, filename=filename, logger=log, revision=args.git_revision
-    )
+    parser = VPPAPI(debug=debug, filename=filename, logger=log, revision=git_revision)
 
     try:
-        if not args.input:
+        if not input_file:
             parsed_objects = parser.parse_fd(sys.stdin, log)
         else:
-            parsed_objects = parser.parse_filename(args.input, log)
+            parsed_objects = parser.parse_filename(input_file, log)
     except ParseError as e:
         print("Parse error: ", e, file=sys.stderr)
         sys.exit(1)
@@ -1274,7 +1250,7 @@ def main():
 
     #
     # Debug
-    if args.debug:
+    if debug:
         import pprint
 
         pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
@@ -1283,14 +1259,71 @@ def main():
         for t in s["types"]:
             pp.pprint([t.name, t.block])
 
-    result = plugin.run(args, filename, s)
+    result = plugin.run(outputdir, filename, s)
     if result:
-        print(result, file=args.output)
+        if isinstance(output, str):
+            with open(output, "w", encoding="UTF-8") as f:
+                print(result, file=f)
+        else:
+            print(result, file=output)
     else:
         log.exception("Running plugin failed: %s %s", filename, result)
         return 1
     return 0
 
 
+def run_kw_vppapigen(kwargs):
+    return run_vppapigen(**kwargs)
+
+
+#
+# Main
+#
+def main():
+    if sys.version_info < (
+        3,
+        5,
+    ):
+        log.exception(
+            "vppapigen requires a supported version of python. "
+            "Please use version 3.5 or greater. "
+            "Using %s",
+            sys.version,
+        )
+        return 1
+
+    cliparser = argparse.ArgumentParser(description="VPP API generator")
+    cliparser.add_argument("--pluginpath", default="")
+    cliparser.add_argument("--includedir", action="append")
+    cliparser.add_argument("--outputdir", action="store")
+    cliparser.add_argument("--input")
+    cliparser.add_argument(
+        "--output",
+        nargs="?",
+        type=argparse.FileType("w", encoding="UTF-8"),
+        default=sys.stdout,
+    )
+
+    cliparser.add_argument("output_module", nargs="?", default="C")
+    cliparser.add_argument("--debug", action="store_true")
+    cliparser.add_argument("--show-name", nargs=1)
+    cliparser.add_argument(
+        "--git-revision", help="Git revision to use for opening files"
+    )
+    args = cliparser.parse_args()
+
+    return run_vppapigen(
+        includedir=args.includedir,
+        debug=args.debug,
+        outputdir=args.outputdir,
+        show_name=args.show_name,
+        input_file=args.input,
+        output_module=args.output_module,
+        pluginpath=args.pluginpath,
+        git_revision=args.git_revision,
+        output=args.output,
+    )
+
+
 if __name__ == "__main__":
     sys.exit(main())
diff --git a/src/tools/vppapigen/vppapigen_c.py b/src/tools/vppapigen/vppapigen_c.py
index fdbb7270a8a..a065653e391 100644
--- a/src/tools/vppapigen/vppapigen_c.py
+++ b/src/tools/vppapigen/vppapigen_c.py
@@ -2010,24 +2010,24 @@ def generate_c_test2_boilerplate(services, defines, module, stream):
 #
 # Plugin entry point
 #
-def run(args, apifilename, s):
+def run(output_dir, apifilename, s):
     """Main plugin entry point."""
     stream = StringIO()
 
-    if not args.outputdir:
+    if not output_dir:
         sys.stderr.write("Missing --outputdir argument")
         return None
 
     basename = os.path.basename(apifilename)
     filename, _ = os.path.splitext(basename)
     modulename = filename.replace(".", "_")
-    filename_enum = os.path.join(args.outputdir + "/" + basename + "_enum.h")
-    filename_types = os.path.join(args.outputdir + "/" + basename + "_types.h")
-    filename_c = os.path.join(args.outputdir + "/" + basename + ".c")
-    filename_c_test = os.path.join(args.outputdir + "/" + basename + "_test.c")
-    filename_c_test2 = os.path.join(args.outputdir + "/" + basename + "_test2.c")
-    filename_c_tojson = os.path.join(args.outputdir + "/" + basename + "_tojson.h")
-    filename_c_fromjson = os.path.join(args.outputdir + "/" + basename + "_fromjson.h")
+    filename_enum = os.path.join(output_dir + "/" + basename + "_enum.h")
+    filename_types = os.path.join(output_dir + "/" + basename + "_types.h")
+    filename_c = os.path.join(output_dir + "/" + basename + ".c")
+    filename_c_test = os.path.join(output_dir + "/" + basename + "_test.c")
+    filename_c_test2 = os.path.join(output_dir + "/" + basename + "_test2.c")
+    filename_c_tojson = os.path.join(output_dir + "/" + basename + "_tojson.h")
+    filename_c_fromjson = os.path.join(output_dir + "/" + basename + "_fromjson.h")
 
     # Generate separate types file
     st = StringIO()
diff --git a/src/tools/vppapigen/vppapigen_crc.py b/src/tools/vppapigen/vppapigen_crc.py
index 525f6c07efc..f7e8296af3e 100644
--- a/src/tools/vppapigen/vppapigen_crc.py
+++ b/src/tools/vppapigen/vppapigen_crc.py
@@ -7,7 +7,7 @@ process_imports = True
 #
 # Plugin entry point
 #
-def run(args, input_filename, s):
+def run(output_dir, input_filename, s):
     j = {}
     major = 0
     minor = 0
diff --git a/src/tools/vppapigen/vppapigen_json.py b/src/tools/vppapigen/vppapigen_json.py
index 695b8cc7aa2..91334468503 100644
--- a/src/tools/vppapigen/vppapigen_json.py
+++ b/src/tools/vppapigen/vppapigen_json.py
@@ -86,7 +86,7 @@ def walk_defs(s, is_message=False):
 #
 # Plugin entry point
 #
-def run(args, filename, s):
+def run(output_dir, filename, s):
     j = {}
 
     j["types"] = walk_defs([o for o in s["types"] if o.__class__.__name__ == "Typedef"])