summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorNathan Skrzypczak <nathan.skrzypczak@gmail.com>2022-06-16 17:00:02 +0200
committerBeno�t Ganne <bganne@cisco.com>2022-08-05 08:33:10 +0000
commit1b299fa46707aa29da6f19f909fcfe354996b8cc (patch)
treef1f60106a3e25cd4a802416acf6c9ea92354a776 /src
parent7ea7ab5f215a95dbc1a38acc03b7fea6d3dbedcf (diff)
vppapigen: make json in parallel
Type: improvement This patches makes the make json-api-files run in parallel in the same python runtime. Default number of workers is 8, and run time goes from ~20s to ~2s on average. Change-Id: Id8cff013889db2671f6b6b4af9a019460c656f81 Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>
Diffstat (limited to 'src')
-rwxr-xr-xsrc/tools/vppapigen/generate_json.py69
-rwxr-xr-xsrc/tools/vppapigen/vppapigen.py149
-rw-r--r--src/tools/vppapigen/vppapigen_c.py18
-rw-r--r--src/tools/vppapigen/vppapigen_crc.py2
-rw-r--r--src/tools/vppapigen/vppapigen_json.py2
5 files changed, 140 insertions, 100 deletions
diff --git a/src/tools/vppapigen/generate_json.py b/src/tools/vppapigen/generate_json.py
index e8041c5a3eb..610f84f5533 100755
--- a/src/tools/vppapigen/generate_json.py
+++ b/src/tools/vppapigen/generate_json.py
@@ -16,15 +16,15 @@
import argparse
import pathlib
import subprocess
+import vppapigen
+import os
+from multiprocessing import Pool
BASE_DIR = (
subprocess.check_output("git rev-parse --show-toplevel", shell=True)
.strip()
.decode()
)
-vppapigen_bin = pathlib.Path(
- "%s/src/tools/vppapigen/vppapigen.py" % BASE_DIR
-).as_posix()
src_dir_depth = 3
output_path = pathlib.Path(
@@ -55,32 +55,14 @@ def api_files(src_dir):
return [x for x in api_search_globs(src_dir)]
-def vppapigen(vppapigen_bin, output_path, src_dir, src_file):
- try:
- subprocess.check_output(
- [
- vppapigen_bin,
- "--includedir",
- src_dir.as_posix(),
- "--input",
- src_file.as_posix(),
- "JSON",
- "--output",
- "%s/%s/%s.json"
- % (
- output_path,
- output_dir_map[
- src_file.as_posix().split("/")[
- src_dir_depth + BASE_DIR.count("/") - 1
- ]
- ],
- src_file.name,
- ),
- ]
- )
- except KeyError:
- print("src_file: %s" % src_file)
- raise
+def get_n_parallel(n_parallel):
+ if n_parallel == 0:
+ n_parallel = os.environ.get("MAKE_PARALLEL_JOBS", os.cpu_count())
+ try:
+ n_parallel = int(n_parallel)
+ except ValueError:
+ return os.cpu_count()
+ return n_parallel or os.cpu_count()
def main():
@@ -88,11 +70,15 @@ def main():
cliparser.add_argument("--srcdir", action="store", default="%s/src" % BASE_DIR),
cliparser.add_argument("--output", action="store", help="directory to store files"),
cliparser.add_argument(
+ "--parallel", type=int, default=0, help="Number of parallel processes"
+ ),
+ cliparser.add_argument(
"--debug-target",
action="store_true",
default=False,
help="'True' if -debug target",
),
+
args = cliparser.parse_args()
src_dir = pathlib.Path(args.srcdir)
@@ -109,8 +95,29 @@ def main():
for f in output_dir.glob("**/*.api.json"):
f.unlink()
- for f in api_files(src_dir):
- vppapigen(vppapigen_bin, output_dir, src_dir, f)
+ with Pool(get_n_parallel(args.parallel)) as p:
+ p.map(
+ vppapigen.run_kw_vppapigen,
+ [
+ {
+ "output": "%s/%s/%s.json"
+ % (
+ output_path,
+ output_dir_map[
+ f.as_posix().split("/")[
+ src_dir_depth + BASE_DIR.count("/") - 1
+ ]
+ ],
+ f.name,
+ ),
+ "input_file": f.as_posix(),
+ "includedir": [src_dir.as_posix()],
+ "output_module": "JSON",
+ }
+ for f in api_files(src_dir)
+ ],
+ )
+
print("json files written to: %s/." % output_dir)
diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py
index 944bf5e7135..9abc5362d80 100755
--- a/src/tools/vppapigen/vppapigen.py
+++ b/src/tools/vppapigen/vppapigen.py
@@ -1152,55 +1152,35 @@ def foldup_crcs(s):
f.crc = foldup_blocks(f.block, binascii.crc32(f.crc) & 0xFFFFFFFF)
-#
-# Main
-#
-def main():
- if sys.version_info < (
- 3,
- 5,
- ):
- log.exception(
- "vppapigen requires a supported version of python. "
- "Please use version 3.5 or greater. "
- "Using %s",
- sys.version,
- )
- return 1
-
- cliparser = argparse.ArgumentParser(description="VPP API generator")
- cliparser.add_argument("--pluginpath", default="")
- cliparser.add_argument("--includedir", action="append")
- cliparser.add_argument("--outputdir", action="store")
- cliparser.add_argument("--input")
- cliparser.add_argument(
- "--output",
- nargs="?",
- type=argparse.FileType("w", encoding="UTF-8"),
- default=sys.stdout,
- )
-
- cliparser.add_argument("output_module", nargs="?", default="C")
- cliparser.add_argument("--debug", action="store_true")
- cliparser.add_argument("--show-name", nargs=1)
- cliparser.add_argument(
- "--git-revision", help="Git revision to use for opening files"
- )
- args = cliparser.parse_args()
-
- dirlist_add(args.includedir)
- if not args.debug:
+def run_vppapigen(
+ input_file=None,
+ output=sys.stdout,
+ includedir=None,
+ debug=False,
+ show_name=None,
+ output_module="C",
+ outputdir=None,
+ pluginpath="",
+ git_revision=None,
+):
+ # reset globals
+ dirlist.clear()
+ global_types.clear()
+ seen_imports.clear()
+
+ dirlist_add(includedir)
+ if not debug:
sys.excepthook = exception_handler
# Filename
- if args.show_name:
- filename = args.show_name[0]
- elif args.input:
- filename = args.input
+ if show_name:
+ filename = show_name[0]
+ elif input_file:
+ filename = input_file
else:
filename = ""
- if args.debug:
+ if debug:
logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
else:
logging.basicConfig()
@@ -1212,39 +1192,35 @@ def main():
# Default path
pluginpath = ""
- if not args.pluginpath:
+ if not pluginpath:
cand = []
cand.append(os.path.dirname(os.path.realpath(__file__)))
cand.append(os.path.dirname(os.path.realpath(__file__)) + "/../share/vpp/")
for c in cand:
c += "/"
- if os.path.isfile(
- "{}vppapigen_{}.py".format(c, args.output_module.lower())
- ):
+ if os.path.isfile("{}vppapigen_{}.py".format(c, output_module.lower())):
pluginpath = c
break
else:
- pluginpath = args.pluginpath + "/"
+ pluginpath = pluginpath + "/"
if pluginpath == "":
log.exception("Output plugin not found")
return 1
- module_path = "{}vppapigen_{}.py".format(pluginpath, args.output_module.lower())
+ module_path = "{}vppapigen_{}.py".format(pluginpath, output_module.lower())
try:
- plugin = SourceFileLoader(args.output_module, module_path).load_module()
+ plugin = SourceFileLoader(output_module, module_path).load_module()
except Exception as err:
log.exception("Error importing output plugin: %s, %s", module_path, err)
return 1
- parser = VPPAPI(
- debug=args.debug, filename=filename, logger=log, revision=args.git_revision
- )
+ parser = VPPAPI(debug=debug, filename=filename, logger=log, revision=git_revision)
try:
- if not args.input:
+ if not input_file:
parsed_objects = parser.parse_fd(sys.stdin, log)
else:
- parsed_objects = parser.parse_filename(args.input, log)
+ parsed_objects = parser.parse_filename(input_file, log)
except ParseError as e:
print("Parse error: ", e, file=sys.stderr)
sys.exit(1)
@@ -1274,7 +1250,7 @@ def main():
#
# Debug
- if args.debug:
+ if debug:
import pprint
pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
@@ -1283,14 +1259,71 @@ def main():
for t in s["types"]:
pp.pprint([t.name, t.block])
- result = plugin.run(args, filename, s)
+ result = plugin.run(outputdir, filename, s)
if result:
- print(result, file=args.output)
+ if isinstance(output, str):
+ with open(output, "w", encoding="UTF-8") as f:
+ print(result, file=f)
+ else:
+ print(result, file=output)
else:
log.exception("Running plugin failed: %s %s", filename, result)
return 1
return 0
+def run_kw_vppapigen(kwargs):
+ return run_vppapigen(**kwargs)
+
+
+#
+# Main
+#
+def main():
+ if sys.version_info < (
+ 3,
+ 5,
+ ):
+ log.exception(
+ "vppapigen requires a supported version of python. "
+ "Please use version 3.5 or greater. "
+ "Using %s",
+ sys.version,
+ )
+ return 1
+
+ cliparser = argparse.ArgumentParser(description="VPP API generator")
+ cliparser.add_argument("--pluginpath", default="")
+ cliparser.add_argument("--includedir", action="append")
+ cliparser.add_argument("--outputdir", action="store")
+ cliparser.add_argument("--input")
+ cliparser.add_argument(
+ "--output",
+ nargs="?",
+ type=argparse.FileType("w", encoding="UTF-8"),
+ default=sys.stdout,
+ )
+
+ cliparser.add_argument("output_module", nargs="?", default="C")
+ cliparser.add_argument("--debug", action="store_true")
+ cliparser.add_argument("--show-name", nargs=1)
+ cliparser.add_argument(
+ "--git-revision", help="Git revision to use for opening files"
+ )
+ args = cliparser.parse_args()
+
+ return run_vppapigen(
+ includedir=args.includedir,
+ debug=args.debug,
+ outputdir=args.outputdir,
+ show_name=args.show_name,
+ input_file=args.input,
+ output_module=args.output_module,
+ pluginpath=args.pluginpath,
+ git_revision=args.git_revision,
+ output=args.output,
+ )
+
+
if __name__ == "__main__":
sys.exit(main())
diff --git a/src/tools/vppapigen/vppapigen_c.py b/src/tools/vppapigen/vppapigen_c.py
index fdbb7270a8a..a065653e391 100644
--- a/src/tools/vppapigen/vppapigen_c.py
+++ b/src/tools/vppapigen/vppapigen_c.py
@@ -2010,24 +2010,24 @@ def generate_c_test2_boilerplate(services, defines, module, stream):
#
# Plugin entry point
#
-def run(args, apifilename, s):
+def run(output_dir, apifilename, s):
"""Main plugin entry point."""
stream = StringIO()
- if not args.outputdir:
+ if not output_dir:
sys.stderr.write("Missing --outputdir argument")
return None
basename = os.path.basename(apifilename)
filename, _ = os.path.splitext(basename)
modulename = filename.replace(".", "_")
- filename_enum = os.path.join(args.outputdir + "/" + basename + "_enum.h")
- filename_types = os.path.join(args.outputdir + "/" + basename + "_types.h")
- filename_c = os.path.join(args.outputdir + "/" + basename + ".c")
- filename_c_test = os.path.join(args.outputdir + "/" + basename + "_test.c")
- filename_c_test2 = os.path.join(args.outputdir + "/" + basename + "_test2.c")
- filename_c_tojson = os.path.join(args.outputdir + "/" + basename + "_tojson.h")
- filename_c_fromjson = os.path.join(args.outputdir + "/" + basename + "_fromjson.h")
+ filename_enum = os.path.join(output_dir + "/" + basename + "_enum.h")
+ filename_types = os.path.join(output_dir + "/" + basename + "_types.h")
+ filename_c = os.path.join(output_dir + "/" + basename + ".c")
+ filename_c_test = os.path.join(output_dir + "/" + basename + "_test.c")
+ filename_c_test2 = os.path.join(output_dir + "/" + basename + "_test2.c")
+ filename_c_tojson = os.path.join(output_dir + "/" + basename + "_tojson.h")
+ filename_c_fromjson = os.path.join(output_dir + "/" + basename + "_fromjson.h")
# Generate separate types file
st = StringIO()
diff --git a/src/tools/vppapigen/vppapigen_crc.py b/src/tools/vppapigen/vppapigen_crc.py
index 525f6c07efc..f7e8296af3e 100644
--- a/src/tools/vppapigen/vppapigen_crc.py
+++ b/src/tools/vppapigen/vppapigen_crc.py
@@ -7,7 +7,7 @@ process_imports = True
#
# Plugin entry point
#
-def run(args, input_filename, s):
+def run(output_dir, input_filename, s):
j = {}
major = 0
minor = 0
diff --git a/src/tools/vppapigen/vppapigen_json.py b/src/tools/vppapigen/vppapigen_json.py
index 695b8cc7aa2..91334468503 100644
--- a/src/tools/vppapigen/vppapigen_json.py
+++ b/src/tools/vppapigen/vppapigen_json.py
@@ -86,7 +86,7 @@ def walk_defs(s, is_message=False):
#
# Plugin entry point
#
-def run(args, filename, s):
+def run(output_dir, filename, s):
j = {}
j["types"] = walk_defs([o for o in s["types"] if o.__class__.__name__ == "Typedef"])