diff options
author | Nathan Skrzypczak <nathan.skrzypczak@gmail.com> | 2022-06-16 17:00:02 +0200 |
---|---|---|
committer | Beno�t Ganne <bganne@cisco.com> | 2022-08-05 08:33:10 +0000 |
commit | 1b299fa46707aa29da6f19f909fcfe354996b8cc (patch) | |
tree | f1f60106a3e25cd4a802416acf6c9ea92354a776 /src/tools/vppapigen/generate_json.py | |
parent | 7ea7ab5f215a95dbc1a38acc03b7fea6d3dbedcf (diff) |
vppapigen: make json in parallel
Type: improvement
This patches makes the make json-api-files
run in parallel in the same python runtime.
Default number of workers is 8, and run time
goes from ~20s to ~2s on average.
Change-Id: Id8cff013889db2671f6b6b4af9a019460c656f81
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>
Diffstat (limited to 'src/tools/vppapigen/generate_json.py')
-rwxr-xr-x | src/tools/vppapigen/generate_json.py | 69 |
1 files changed, 38 insertions, 31 deletions
diff --git a/src/tools/vppapigen/generate_json.py b/src/tools/vppapigen/generate_json.py index e8041c5a3eb..610f84f5533 100755 --- a/src/tools/vppapigen/generate_json.py +++ b/src/tools/vppapigen/generate_json.py @@ -16,15 +16,15 @@ import argparse import pathlib import subprocess +import vppapigen +import os +from multiprocessing import Pool BASE_DIR = ( subprocess.check_output("git rev-parse --show-toplevel", shell=True) .strip() .decode() ) -vppapigen_bin = pathlib.Path( - "%s/src/tools/vppapigen/vppapigen.py" % BASE_DIR -).as_posix() src_dir_depth = 3 output_path = pathlib.Path( @@ -55,32 +55,14 @@ def api_files(src_dir): return [x for x in api_search_globs(src_dir)] -def vppapigen(vppapigen_bin, output_path, src_dir, src_file): - try: - subprocess.check_output( - [ - vppapigen_bin, - "--includedir", - src_dir.as_posix(), - "--input", - src_file.as_posix(), - "JSON", - "--output", - "%s/%s/%s.json" - % ( - output_path, - output_dir_map[ - src_file.as_posix().split("/")[ - src_dir_depth + BASE_DIR.count("/") - 1 - ] - ], - src_file.name, - ), - ] - ) - except KeyError: - print("src_file: %s" % src_file) - raise +def get_n_parallel(n_parallel): + if n_parallel == 0: + n_parallel = os.environ.get("MAKE_PARALLEL_JOBS", os.cpu_count()) + try: + n_parallel = int(n_parallel) + except ValueError: + return os.cpu_count() + return n_parallel or os.cpu_count() def main(): @@ -88,11 +70,15 @@ def main(): cliparser.add_argument("--srcdir", action="store", default="%s/src" % BASE_DIR), cliparser.add_argument("--output", action="store", help="directory to store files"), cliparser.add_argument( + "--parallel", type=int, default=0, help="Number of parallel processes" + ), + cliparser.add_argument( "--debug-target", action="store_true", default=False, help="'True' if -debug target", ), + args = cliparser.parse_args() src_dir = pathlib.Path(args.srcdir) @@ -109,8 +95,29 @@ def main(): for f in output_dir.glob("**/*.api.json"): f.unlink() - for f in api_files(src_dir): - vppapigen(vppapigen_bin, output_dir, src_dir, f) + with Pool(get_n_parallel(args.parallel)) as p: + p.map( + vppapigen.run_kw_vppapigen, + [ + { + "output": "%s/%s/%s.json" + % ( + output_path, + output_dir_map[ + f.as_posix().split("/")[ + src_dir_depth + BASE_DIR.count("/") - 1 + ] + ], + f.name, + ), + "input_file": f.as_posix(), + "includedir": [src_dir.as_posix()], + "output_module": "JSON", + } + for f in api_files(src_dir) + ], + ) + print("json files written to: %s/." % output_dir) |