vppapigen: make json in parallel

Type: improvement

This patches makes the make json-api-files
run in parallel in the same python runtime.

Default number of workers is 8, and run time
goes from ~20s to ~2s on average.

Change-Id: Id8cff013889db2671f6b6b4af9a019460c656f81
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>

1 files changed, 91 insertions, 58 deletions

diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py
index 944bf5e7135..9abc5362d80 100755
--- a/src/tools/vppapigen/vppapigen.py
+++ b/src/tools/vppapigen/vppapigen.py
@@ -1152,55 +1152,35 @@ def foldup_crcs(s):
         f.crc = foldup_blocks(f.block, binascii.crc32(f.crc) & 0xFFFFFFFF)
 
 
-#
-# Main
-#
-def main():
-    if sys.version_info < (
-        3,
-        5,
-    ):
-        log.exception(
-            "vppapigen requires a supported version of python. "
-            "Please use version 3.5 or greater. "
-            "Using %s",
-            sys.version,
-        )
-        return 1
-
-    cliparser = argparse.ArgumentParser(description="VPP API generator")
-    cliparser.add_argument("--pluginpath", default="")
-    cliparser.add_argument("--includedir", action="append")
-    cliparser.add_argument("--outputdir", action="store")
-    cliparser.add_argument("--input")
-    cliparser.add_argument(
-        "--output",
-        nargs="?",
-        type=argparse.FileType("w", encoding="UTF-8"),
-        default=sys.stdout,
-    )
-
-    cliparser.add_argument("output_module", nargs="?", default="C")
-    cliparser.add_argument("--debug", action="store_true")
-    cliparser.add_argument("--show-name", nargs=1)
-    cliparser.add_argument(
-        "--git-revision", help="Git revision to use for opening files"
-    )
-    args = cliparser.parse_args()
-
-    dirlist_add(args.includedir)
-    if not args.debug:
+def run_vppapigen(
+    input_file=None,
+    output=sys.stdout,
+    includedir=None,
+    debug=False,
+    show_name=None,
+    output_module="C",
+    outputdir=None,
+    pluginpath="",
+    git_revision=None,
+):
+    # reset globals
+    dirlist.clear()
+    global_types.clear()
+    seen_imports.clear()
+
+    dirlist_add(includedir)
+    if not debug:
         sys.excepthook = exception_handler
 
     # Filename
-    if args.show_name:
-        filename = args.show_name[0]
-    elif args.input:
-        filename = args.input
+    if show_name:
+        filename = show_name[0]
+    elif input_file:
+        filename = input_file
     else:
         filename = ""
 
-    if args.debug:
+    if debug:
         logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
     else:
         logging.basicConfig()
@@ -1212,39 +1192,35 @@ def main():
 
     # Default path
     pluginpath = ""
-    if not args.pluginpath:
+    if not pluginpath:
         cand = []
         cand.append(os.path.dirname(os.path.realpath(__file__)))
         cand.append(os.path.dirname(os.path.realpath(__file__)) + "/../share/vpp/")
         for c in cand:
             c += "/"
-            if os.path.isfile(
-                "{}vppapigen_{}.py".format(c, args.output_module.lower())
-            ):
+            if os.path.isfile("{}vppapigen_{}.py".format(c, output_module.lower())):
                 pluginpath = c
                 break
     else:
-        pluginpath = args.pluginpath + "/"
+        pluginpath = pluginpath + "/"
     if pluginpath == "":
         log.exception("Output plugin not found")
         return 1
-    module_path = "{}vppapigen_{}.py".format(pluginpath, args.output_module.lower())
+    module_path = "{}vppapigen_{}.py".format(pluginpath, output_module.lower())
 
     try:
-        plugin = SourceFileLoader(args.output_module, module_path).load_module()
+        plugin = SourceFileLoader(output_module, module_path).load_module()
     except Exception as err:
         log.exception("Error importing output plugin: %s, %s", module_path, err)
         return 1
 
-    parser = VPPAPI(
-        debug=args.debug, filename=filename, logger=log, revision=args.git_revision
-    )
+    parser = VPPAPI(debug=debug, filename=filename, logger=log, revision=git_revision)
 
     try:
-        if not args.input:
+        if not input_file:
             parsed_objects = parser.parse_fd(sys.stdin, log)
         else:
-            parsed_objects = parser.parse_filename(args.input, log)
+            parsed_objects = parser.parse_filename(input_file, log)
     except ParseError as e:
         print("Parse error: ", e, file=sys.stderr)
         sys.exit(1)
@@ -1274,7 +1250,7 @@ def main():
 
     #
     # Debug
-    if args.debug:
+    if debug:
         import pprint
 
         pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
@@ -1283,14 +1259,71 @@ def main():
         for t in s["types"]:
             pp.pprint([t.name, t.block])
 
-    result = plugin.run(args, filename, s)
+    result = plugin.run(outputdir, filename, s)
     if result:
-        print(result, file=args.output)
+        if isinstance(output, str):
+            with open(output, "w", encoding="UTF-8") as f:
+                print(result, file=f)
+        else:
+            print(result, file=output)
     else:
         log.exception("Running plugin failed: %s %s", filename, result)
         return 1
     return 0
 
 
+def run_kw_vppapigen(kwargs):
+    return run_vppapigen(**kwargs)
+
+
+#
+# Main
+#
+def main():
+    if sys.version_info < (
+        3,
+        5,
+    ):
+        log.exception(
+            "vppapigen requires a supported version of python. "
+            "Please use version 3.5 or greater. "
+            "Using %s",
+            sys.version,
+        )
+        return 1
+
+    cliparser = argparse.ArgumentParser(description="VPP API generator")
+    cliparser.add_argument("--pluginpath", default="")
+    cliparser.add_argument("--includedir", action="append")
+    cliparser.add_argument("--outputdir", action="store")
+    cliparser.add_argument("--input")
+    cliparser.add_argument(
+        "--output",
+        nargs="?",
+        type=argparse.FileType("w", encoding="UTF-8"),
+        default=sys.stdout,
+    )
+
+    cliparser.add_argument("output_module", nargs="?", default="C")
+    cliparser.add_argument("--debug", action="store_true")
+    cliparser.add_argument("--show-name", nargs=1)
+    cliparser.add_argument(
+        "--git-revision", help="Git revision to use for opening files"
+    )
+    args = cliparser.parse_args()
+
+    return run_vppapigen(
+        includedir=args.includedir,
+        debug=args.debug,
+        outputdir=args.outputdir,
+        show_name=args.show_name,
+        input_file=args.input,
+        output_module=args.output_module,
+        pluginpath=args.pluginpath,
+        git_revision=args.git_revision,
+        output=args.output,
+    )
+
+
 if __name__ == "__main__":
     sys.exit(main())