diff options
| -rwxr-xr-x | src/tools/vppapigen/generate_json.py | 69 | ||||
| -rwxr-xr-x | src/tools/vppapigen/vppapigen.py | 149 | ||||
| -rw-r--r-- | src/tools/vppapigen/vppapigen_c.py | 18 | ||||
| -rw-r--r-- | src/tools/vppapigen/vppapigen_crc.py | 2 | ||||
| -rw-r--r-- | src/tools/vppapigen/vppapigen_json.py | 2 |
5 files changed, 140 insertions, 100 deletions
diff --git a/src/tools/vppapigen/generate_json.py b/src/tools/vppapigen/generate_json.py index e8041c5a3eb..610f84f5533 100755 --- a/src/tools/vppapigen/generate_json.py +++ b/src/tools/vppapigen/generate_json.py @@ -16,15 +16,15 @@ import argparse import pathlib import subprocess +import vppapigen +import os +from multiprocessing import Pool BASE_DIR = ( subprocess.check_output("git rev-parse --show-toplevel", shell=True) .strip() .decode() ) -vppapigen_bin = pathlib.Path( - "%s/src/tools/vppapigen/vppapigen.py" % BASE_DIR -).as_posix() src_dir_depth = 3 output_path = pathlib.Path( @@ -55,32 +55,14 @@ def api_files(src_dir): return [x for x in api_search_globs(src_dir)] -def vppapigen(vppapigen_bin, output_path, src_dir, src_file): - try: - subprocess.check_output( - [ - vppapigen_bin, - "--includedir", - src_dir.as_posix(), - "--input", - src_file.as_posix(), - "JSON", - "--output", - "%s/%s/%s.json" - % ( - output_path, - output_dir_map[ - src_file.as_posix().split("/")[ - src_dir_depth + BASE_DIR.count("/") - 1 - ] - ], - src_file.name, - ), - ] - ) - except KeyError: - print("src_file: %s" % src_file) - raise +def get_n_parallel(n_parallel): + if n_parallel == 0: + n_parallel = os.environ.get("MAKE_PARALLEL_JOBS", os.cpu_count()) + try: + n_parallel = int(n_parallel) + except ValueError: + return os.cpu_count() + return n_parallel or os.cpu_count() def main(): @@ -88,11 +70,15 @@ def main(): cliparser.add_argument("--srcdir", action="store", default="%s/src" % BASE_DIR), cliparser.add_argument("--output", action="store", help="directory to store files"), cliparser.add_argument( + "--parallel", type=int, default=0, help="Number of parallel processes" + ), + cliparser.add_argument( "--debug-target", action="store_true", default=False, help="'True' if -debug target", ), + args = cliparser.parse_args() src_dir = pathlib.Path(args.srcdir) @@ -109,8 +95,29 @@ def main(): for f in output_dir.glob("**/*.api.json"): f.unlink() - for f in api_files(src_dir): - vppapigen(vppapigen_bin, output_dir, src_dir, f) + with Pool(get_n_parallel(args.parallel)) as p: + p.map( + vppapigen.run_kw_vppapigen, + [ + { + "output": "%s/%s/%s.json" + % ( + output_path, + output_dir_map[ + f.as_posix().split("/")[ + src_dir_depth + BASE_DIR.count("/") - 1 + ] + ], + f.name, + ), + "input_file": f.as_posix(), + "includedir": [src_dir.as_posix()], + "output_module": "JSON", + } + for f in api_files(src_dir) + ], + ) + print("json files written to: %s/." % output_dir) diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py index 944bf5e7135..9abc5362d80 100755 --- a/src/tools/vppapigen/vppapigen.py +++ b/src/tools/vppapigen/vppapigen.py @@ -1152,55 +1152,35 @@ def foldup_crcs(s): f.crc = foldup_blocks(f.block, binascii.crc32(f.crc) & 0xFFFFFFFF) -# -# Main -# -def main(): - if sys.version_info < ( - 3, - 5, - ): - log.exception( - "vppapigen requires a supported version of python. " - "Please use version 3.5 or greater. " - "Using %s", - sys.version, - ) - return 1 - - cliparser = argparse.ArgumentParser(description="VPP API generator") - cliparser.add_argument("--pluginpath", default="") - cliparser.add_argument("--includedir", action="append") - cliparser.add_argument("--outputdir", action="store") - cliparser.add_argument("--input") - cliparser.add_argument( - "--output", - nargs="?", - type=argparse.FileType("w", encoding="UTF-8"), - default=sys.stdout, - ) - - cliparser.add_argument("output_module", nargs="?", default="C") - cliparser.add_argument("--debug", action="store_true") - cliparser.add_argument("--show-name", nargs=1) - cliparser.add_argument( - "--git-revision", help="Git revision to use for opening files" - ) - args = cliparser.parse_args() - - dirlist_add(args.includedir) - if not args.debug: +def run_vppapigen( + input_file=None, + output=sys.stdout, + includedir=None, + debug=False, + show_name=None, + output_module="C", + outputdir=None, + pluginpath="", + git_revision=None, +): + # reset globals + dirlist.clear() + global_types.clear() + seen_imports.clear() + + dirlist_add(includedir) + if not debug: sys.excepthook = exception_handler # Filename - if args.show_name: - filename = args.show_name[0] - elif args.input: - filename = args.input + if show_name: + filename = show_name[0] + elif input_file: + filename = input_file else: filename = "" - if args.debug: + if debug: logging.basicConfig(stream=sys.stdout, level=logging.WARNING) else: logging.basicConfig() @@ -1212,39 +1192,35 @@ def main(): # Default path pluginpath = "" - if not args.pluginpath: + if not pluginpath: cand = [] cand.append(os.path.dirname(os.path.realpath(__file__))) cand.append(os.path.dirname(os.path.realpath(__file__)) + "/../share/vpp/") for c in cand: c += "/" - if os.path.isfile( - "{}vppapigen_{}.py".format(c, args.output_module.lower()) - ): + if os.path.isfile("{}vppapigen_{}.py".format(c, output_module.lower())): pluginpath = c break else: - pluginpath = args.pluginpath + "/" + pluginpath = pluginpath + "/" if pluginpath == "": log.exception("Output plugin not found") return 1 - module_path = "{}vppapigen_{}.py".format(pluginpath, args.output_module.lower()) + module_path = "{}vppapigen_{}.py".format(pluginpath, output_module.lower()) try: - plugin = SourceFileLoader(args.output_module, module_path).load_module() + plugin = SourceFileLoader(output_module, module_path).load_module() except Exception as err: log.exception("Error importing output plugin: %s, %s", module_path, err) return 1 - parser = VPPAPI( - debug=args.debug, filename=filename, logger=log, revision=args.git_revision - ) + parser = VPPAPI(debug=debug, filename=filename, logger=log, revision=git_revision) try: - if not args.input: + if not input_file: parsed_objects = parser.parse_fd(sys.stdin, log) else: - parsed_objects = parser.parse_filename(args.input, log) + parsed_objects = parser.parse_filename(input_file, log) except ParseError as e: print("Parse error: ", e, file=sys.stderr) sys.exit(1) @@ -1274,7 +1250,7 @@ def main(): # # Debug - if args.debug: + if debug: import pprint pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr) @@ -1283,14 +1259,71 @@ def main(): for t in s["types"]: pp.pprint([t.name, t.block]) - result = plugin.run(args, filename, s) + result = plugin.run(outputdir, filename, s) if result: - print(result, file=args.output) + if isinstance(output, str): + with open(output, "w", encoding="UTF-8") as f: + print(result, file=f) + else: + print(result, file=output) else: log.exception("Running plugin failed: %s %s", filename, result) return 1 return 0 +def run_kw_vppapigen(kwargs): + return run_vppapigen(**kwargs) + + +# +# Main +# +def main(): + if sys.version_info < ( + 3, + 5, + ): + log.exception( + "vppapigen requires a supported version of python. " + "Please use version 3.5 or greater. " + "Using %s", + sys.version, + ) + return 1 + + cliparser = argparse.ArgumentParser(description="VPP API generator") + cliparser.add_argument("--pluginpath", default="") + cliparser.add_argument("--includedir", action="append") + cliparser.add_argument("--outputdir", action="store") + cliparser.add_argument("--input") + cliparser.add_argument( + "--output", + nargs="?", + type=argparse.FileType("w", encoding="UTF-8"), + default=sys.stdout, + ) + + cliparser.add_argument("output_module", nargs="?", default="C") + cliparser.add_argument("--debug", action="store_true") + cliparser.add_argument("--show-name", nargs=1) + cliparser.add_argument( + "--git-revision", help="Git revision to use for opening files" + ) + args = cliparser.parse_args() + + return run_vppapigen( + includedir=args.includedir, + debug=args.debug, + outputdir=args.outputdir, + show_name=args.show_name, + input_file=args.input, + output_module=args.output_module, + pluginpath=args.pluginpath, + git_revision=args.git_revision, + output=args.output, + ) + + if __name__ == "__main__": sys.exit(main()) diff --git a/src/tools/vppapigen/vppapigen_c.py b/src/tools/vppapigen/vppapigen_c.py index fdbb7270a8a..a065653e391 100644 --- a/src/tools/vppapigen/vppapigen_c.py +++ b/src/tools/vppapigen/vppapigen_c.py @@ -2010,24 +2010,24 @@ def generate_c_test2_boilerplate(services, defines, module, stream): # # Plugin entry point # -def run(args, apifilename, s): +def run(output_dir, apifilename, s): """Main plugin entry point.""" stream = StringIO() - if not args.outputdir: + if not output_dir: sys.stderr.write("Missing --outputdir argument") return None basename = os.path.basename(apifilename) filename, _ = os.path.splitext(basename) modulename = filename.replace(".", "_") - filename_enum = os.path.join(args.outputdir + "/" + basename + "_enum.h") - filename_types = os.path.join(args.outputdir + "/" + basename + "_types.h") - filename_c = os.path.join(args.outputdir + "/" + basename + ".c") - filename_c_test = os.path.join(args.outputdir + "/" + basename + "_test.c") - filename_c_test2 = os.path.join(args.outputdir + "/" + basename + "_test2.c") - filename_c_tojson = os.path.join(args.outputdir + "/" + basename + "_tojson.h") - filename_c_fromjson = os.path.join(args.outputdir + "/" + basename + "_fromjson.h") + filename_enum = os.path.join(output_dir + "/" + basename + "_enum.h") + filename_types = os.path.join(output_dir + "/" + basename + "_types.h") + filename_c = os.path.join(output_dir + "/" + basename + ".c") + filename_c_test = os.path.join(output_dir + "/" + basename + "_test.c") + filename_c_test2 = os.path.join(output_dir + "/" + basename + "_test2.c") + filename_c_tojson = os.path.join(output_dir + "/" + basename + "_tojson.h") + filename_c_fromjson = os.path.join(output_dir + "/" + basename + "_fromjson.h") # Generate separate types file st = StringIO() diff --git a/src/tools/vppapigen/vppapigen_crc.py b/src/tools/vppapigen/vppapigen_crc.py index 525f6c07efc..f7e8296af3e 100644 --- a/src/tools/vppapigen/vppapigen_crc.py +++ b/src/tools/vppapigen/vppapigen_crc.py @@ -7,7 +7,7 @@ process_imports = True # # Plugin entry point # -def run(args, input_filename, s): +def run(output_dir, input_filename, s): j = {} major = 0 minor = 0 diff --git a/src/tools/vppapigen/vppapigen_json.py b/src/tools/vppapigen/vppapigen_json.py index 695b8cc7aa2..91334468503 100644 --- a/src/tools/vppapigen/vppapigen_json.py +++ b/src/tools/vppapigen/vppapigen_json.py @@ -86,7 +86,7 @@ def walk_defs(s, is_message=False): # # Plugin entry point # -def run(args, filename, s): +def run(output_dir, filename, s): j = {} j["types"] = walk_defs([o for o in s["types"] if o.__class__.__name__ == "Typedef"]) |