summaryrefslogtreecommitdiffstats
path: root/src/tools/vppapigen/vppapigen.py
diff options
context:
space:
mode:
authorNathan Skrzypczak <nathan.skrzypczak@gmail.com>2022-06-16 17:00:02 +0200
committerBeno�t Ganne <bganne@cisco.com>2022-08-05 08:33:10 +0000
commit1b299fa46707aa29da6f19f909fcfe354996b8cc (patch)
treef1f60106a3e25cd4a802416acf6c9ea92354a776 /src/tools/vppapigen/vppapigen.py
parent7ea7ab5f215a95dbc1a38acc03b7fea6d3dbedcf (diff)
vppapigen: make json in parallel
Type: improvement This patches makes the make json-api-files run in parallel in the same python runtime. Default number of workers is 8, and run time goes from ~20s to ~2s on average. Change-Id: Id8cff013889db2671f6b6b4af9a019460c656f81 Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>
Diffstat (limited to 'src/tools/vppapigen/vppapigen.py')
-rwxr-xr-xsrc/tools/vppapigen/vppapigen.py149
1 files changed, 91 insertions, 58 deletions
diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py
index 944bf5e7135..9abc5362d80 100755
--- a/src/tools/vppapigen/vppapigen.py
+++ b/src/tools/vppapigen/vppapigen.py
@@ -1152,55 +1152,35 @@ def foldup_crcs(s):
f.crc = foldup_blocks(f.block, binascii.crc32(f.crc) & 0xFFFFFFFF)
-#
-# Main
-#
-def main():
- if sys.version_info < (
- 3,
- 5,
- ):
- log.exception(
- "vppapigen requires a supported version of python. "
- "Please use version 3.5 or greater. "
- "Using %s",
- sys.version,
- )
- return 1
-
- cliparser = argparse.ArgumentParser(description="VPP API generator")
- cliparser.add_argument("--pluginpath", default="")
- cliparser.add_argument("--includedir", action="append")
- cliparser.add_argument("--outputdir", action="store")
- cliparser.add_argument("--input")
- cliparser.add_argument(
- "--output",
- nargs="?",
- type=argparse.FileType("w", encoding="UTF-8"),
- default=sys.stdout,
- )
-
- cliparser.add_argument("output_module", nargs="?", default="C")
- cliparser.add_argument("--debug", action="store_true")
- cliparser.add_argument("--show-name", nargs=1)
- cliparser.add_argument(
- "--git-revision", help="Git revision to use for opening files"
- )
- args = cliparser.parse_args()
-
- dirlist_add(args.includedir)
- if not args.debug:
+def run_vppapigen(
+ input_file=None,
+ output=sys.stdout,
+ includedir=None,
+ debug=False,
+ show_name=None,
+ output_module="C",
+ outputdir=None,
+ pluginpath="",
+ git_revision=None,
+):
+ # reset globals
+ dirlist.clear()
+ global_types.clear()
+ seen_imports.clear()
+
+ dirlist_add(includedir)
+ if not debug:
sys.excepthook = exception_handler
# Filename
- if args.show_name:
- filename = args.show_name[0]
- elif args.input:
- filename = args.input
+ if show_name:
+ filename = show_name[0]
+ elif input_file:
+ filename = input_file
else:
filename = ""
- if args.debug:
+ if debug:
logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
else:
logging.basicConfig()
@@ -1212,39 +1192,35 @@ def main():
# Default path
pluginpath = ""
- if not args.pluginpath:
+ if not pluginpath:
cand = []
cand.append(os.path.dirname(os.path.realpath(__file__)))
cand.append(os.path.dirname(os.path.realpath(__file__)) + "/../share/vpp/")
for c in cand:
c += "/"
- if os.path.isfile(
- "{}vppapigen_{}.py".format(c, args.output_module.lower())
- ):
+ if os.path.isfile("{}vppapigen_{}.py".format(c, output_module.lower())):
pluginpath = c
break
else:
- pluginpath = args.pluginpath + "/"
+ pluginpath = pluginpath + "/"
if pluginpath == "":
log.exception("Output plugin not found")
return 1
- module_path = "{}vppapigen_{}.py".format(pluginpath, args.output_module.lower())
+ module_path = "{}vppapigen_{}.py".format(pluginpath, output_module.lower())
try:
- plugin = SourceFileLoader(args.output_module, module_path).load_module()
+ plugin = SourceFileLoader(output_module, module_path).load_module()
except Exception as err:
log.exception("Error importing output plugin: %s, %s", module_path, err)
return 1
- parser = VPPAPI(
- debug=args.debug, filename=filename, logger=log, revision=args.git_revision
- )
+ parser = VPPAPI(debug=debug, filename=filename, logger=log, revision=git_revision)
try:
- if not args.input:
+ if not input_file:
parsed_objects = parser.parse_fd(sys.stdin, log)
else:
- parsed_objects = parser.parse_filename(args.input, log)
+ parsed_objects = parser.parse_filename(input_file, log)
except ParseError as e:
print("Parse error: ", e, file=sys.stderr)
sys.exit(1)
@@ -1274,7 +1250,7 @@ def main():
#
# Debug
- if args.debug:
+ if debug:
import pprint
pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
@@ -1283,14 +1259,71 @@ def main():
for t in s["types"]:
pp.pprint([t.name, t.block])
- result = plugin.run(args, filename, s)
+ result = plugin.run(outputdir, filename, s)
if result:
- print(result, file=args.output)
+ if isinstance(output, str):
+ with open(output, "w", encoding="UTF-8") as f:
+ print(result, file=f)
+ else:
+ print(result, file=output)
else:
log.exception("Running plugin failed: %s %s", filename, result)
return 1
return 0
+def run_kw_vppapigen(kwargs):
+ return run_vppapigen(**kwargs)
+
+
+#
+# Main
+#
+def main():
+ if sys.version_info < (
+ 3,
+ 5,
+ ):
+ log.exception(
+ "vppapigen requires a supported version of python. "
+ "Please use version 3.5 or greater. "
+ "Using %s",
+ sys.version,
+ )
+ return 1
+
+ cliparser = argparse.ArgumentParser(description="VPP API generator")
+ cliparser.add_argument("--pluginpath", default="")
+ cliparser.add_argument("--includedir", action="append")
+ cliparser.add_argument("--outputdir", action="store")
+ cliparser.add_argument("--input")
+ cliparser.add_argument(
+ "--output",
+ nargs="?",
+ type=argparse.FileType("w", encoding="UTF-8"),
+ default=sys.stdout,
+ )
+
+ cliparser.add_argument("output_module", nargs="?", default="C")
+ cliparser.add_argument("--debug", action="store_true")
+ cliparser.add_argument("--show-name", nargs=1)
+ cliparser.add_argument(
+ "--git-revision", help="Git revision to use for opening files"
+ )
+ args = cliparser.parse_args()
+
+ return run_vppapigen(
+ includedir=args.includedir,
+ debug=args.debug,
+ outputdir=args.outputdir,
+ show_name=args.show_name,
+ input_file=args.input,
+ output_module=args.output_module,
+ pluginpath=args.pluginpath,
+ git_revision=args.git_revision,
+ output=args.output,
+ )
+
+
if __name__ == "__main__":
sys.exit(main())