diff options
48 files changed, 814 insertions, 525 deletions
@@ -262,7 +262,6 @@ help: @echo " startup.conf file is present" @echo " GDB=<path> - gdb binary to use for debugging" @echo " PLATFORM=<name> - target platform. default is vpp" - @echo " TEST=<filter> - apply filter to test set, see test-help" @echo " DPDK_CONFIG=<conf> - add specified dpdk config commands to" @echo " autogenerated startup.conf" @echo " (e.g. \"no-pci\" )" @@ -413,7 +412,6 @@ wipe-release: test-wipe $(BR)/.deps.ok rebuild-release: wipe-release build-release export TEST_DIR ?= $(WS_ROOT)/test -export RND_SEED ?= $(shell python3 -c 'import time; print(time.time())') define test $(if $(filter-out $(2),retest),make -C $(BR) PLATFORM=vpp TAG=$(1) vpp-install,) @@ -427,6 +425,7 @@ define test OS_ID=$(OS_ID) \ RND_SEED=$(RND_SEED) \ CACHE_OUTPUT=$(CACHE_OUTPUT) \ + TAG=$(1) \ $(2) endef @@ -444,12 +443,12 @@ test-gcov: .PHONY: test-all test-all: - $(eval EXTENDED_TESTS=yes) + $(eval EXTENDED_TESTS=1) $(call test,vpp,test) .PHONY: test-all-debug test-all-debug: - $(eval EXTENDED_TESTS=yes) + $(eval EXTENDED_TESTS=1) $(call test,vpp_debug,test) .PHONY: papi-wipe @@ -496,7 +495,7 @@ test-wipe-doc: .PHONY: test-cov test-cov: - $(eval EXTENDED_TESTS=yes) + $(eval EXTENDED_TESTS=1) $(call test,vpp_gcov,cov) .PHONY: test-wipe-cov @@ -529,12 +528,12 @@ retest-debug: .PHONY: retest-all retest-all: - $(eval EXTENDED_TESTS=yes) + $(eval EXTENDED_TESTS=1) $(call test,vpp,retest) .PHONY: retest-all-debug retest-all-debug: - $(eval EXTENDED_TESTS=yes) + $(eval EXTENDED_TESTS=1) $(call test,vpp_debug,retest) .PHONY: test-start-vpp-in-gdb diff --git a/docs/developer/build-run-debug/testing_vpp.rst b/docs/developer/build-run-debug/testing_vpp.rst index ca9a09efb71..28ddb21b122 100644 --- a/docs/developer/build-run-debug/testing_vpp.rst +++ b/docs/developer/build-run-debug/testing_vpp.rst @@ -43,98 +43,85 @@ Current “make test-help” output: :: - $ make test-help - test - build and run (basic) functional tests - test-debug - build and run (basic) functional tests (debug build) - test-all - build and run functional and extended tests - test-all-debug - build and run functional and extended tests (debug build) - retest - run functional tests - retest-debug - run functional tests (debug build) - retest-all - run functional and extended tests - retest-all-debug - run functional and extended tests (debug build) - test-cov - generate code coverage report for test framework - test-gcov - build and run functional tests (gcov build) - test-wipe - wipe (temporary) files generated by unit tests - test-wipe-cov - wipe code coverage report for test framework - test-wipe-doc - wipe documentation for test framework - test-wipe-papi - rebuild vpp_papi sources - test-wipe-all - wipe (temporary) files generated by unit tests, docs, and coverage - test-shell - enter shell with test environment - test-shell-debug - enter shell with test environment (debug build) - test-checkstyle - check PEP8 compliance for test framework - test-refresh-deps - refresh the Python dependencies for the tests - - Arguments controlling test runs: - V=[0|1|2] - set test verbosity level - 0=ERROR, 1=INFO, 2=DEBUG - TEST_JOBS=[<n>|auto] - use at most <n> parallel python processes for test execution, if auto, set to number of available cpus (default: 1) - MAX_VPP_CPUS=[<n>|auto]- use at most <n> cpus for running vpp main and worker threads, if auto, set to number of available cpus (default: auto) - CACHE_OUTPUT=[0|1] - cache VPP stdout/stderr and log as one block after test finishes (default: 1) - FAILFAST=[0|1] - fail fast if 1, complete all tests if 0 - TIMEOUT=<timeout> - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600) - RETRIES=<n> - retry failed tests <n> times - DEBUG=<type> - set VPP debugging kind - DEBUG=core - detect coredump and load it in gdb on crash - DEBUG=gdb - allow easy debugging by printing VPP PID - and waiting for user input before running - and tearing down a testcase - DEBUG=gdbserver - run gdb inside a gdb server, otherwise - same as above - DEBUG=attach - attach test case to already running vpp in gdb (see test-start-vpp-in-gdb) - - STEP=[yes|no] - ease debugging by stepping through a testcase - SANITY=[yes|no] - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes) - EXTENDED_TESTS=[1|y] - used by '[re]test-all' & '[re]test-all-debug' to run extended tests - TEST=<filter> - filter the set of tests: - by file-name - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py - by file-suffix - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py - by wildcard - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*' - e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name - TEST='bfd.*.*' is equivalent to above example of filter by file-suffix - TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class - TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase - TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes - - VARIANT=<variant> - specify which march node variant to unit test - e.g. VARIANT=skx test the skx march variants - e.g. VARIANT=icl test the icl march variants - - COREDUMP_SIZE=<size> - pass <size> as unix { coredump-size <size> } argument to vpp - e.g. COREDUMP_SIZE=4g - COREDUMP_SIZE=unlimited - COREDUMP_COMPRESS=1 - compress core files if not debugging them - EXTERN_TESTS=<path> - path to out-of-tree test_<name>.py files containing test cases - EXTERN_PLUGINS=<path> - path to out-of-tree plugins to be loaded by vpp under test - EXTERN_COV_DIR=<path> - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report - - PROFILE=1 - enable profiling of test framework via cProfile module - PROFILE_SORT_BY=opt - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime) - PROFILE_OUTPUT=file - output profiling info to file - use absolute path (default: stdout) - - TEST_DEBUG=1 - turn on debugging of the test framework itself (expert) - - SKIP_AARCH64=1 - skip tests that are failing on the ARM platorm in FD.io CI - - RND_SEED=seed - Seed RND with given seed - - Starting VPP in GDB for use with DEBUG=attach: - - test-start-vpp-in-gdb - start VPP in gdb (release) - test-start-vpp-debug-in-gdb - start VPP in gdb (debug) - - Arguments controlling VPP in GDB runs: - - VPP_IN_GDB_TMP_DIR - specify directory to run VPP IN (default: /tmp/unittest-attach-gdb) - VPP_IN_GDB_NO_RMDIR=0 - don't remove existing tmp dir but fail instead - VPP_IN_GDB_CMDLINE=1 - add 'interactive' to VPP arguments to run with command line - - Creating test documentation - test-doc - generate documentation for test framework - test-wipe-doc - wipe documentation for test framework - - Creating test code coverage report - test-cov - generate code coverage report for test framework - test-wipe-cov - wipe code coverage report for test framework - - Verifying code-style - test-checkstyle - check PEP8 compliance + $ make test-help + Running tests: + + test - build and run (basic) functional tests + test-debug - build and run (basic) functional tests (debug build) + test-all - build and run functional and extended tests + test-all-debug - build and run functional and extended tests (debug build) + retest - run functional tests + retest-debug - run functional tests (debug build) + retest-all - run functional and extended tests + retest-all-debug - run functional and extended tests (debug build) + test-cov - generate code coverage report for test framework + test-gcov - build and run functional tests (gcov build) + test-wipe - wipe (temporary) files generated by unit tests + test-wipe-cov - wipe code coverage report for test framework + test-wipe-papi - rebuild vpp_papi sources + test-wipe-all - wipe (temporary) files generated by unit tests, and coverage + test-shell - enter shell with test environment + test-shell-debug - enter shell with test environment (debug build) + test-checkstyle - check PEP8 compliance for test framework + test-refresh-deps - refresh the Python dependencies for the tests + + Arguments controlling test runs: + + V=[0|1|2] - set test verbosity level + 0=ERROR, 1=INFO, 2=DEBUG + TEST_JOBS=[<n>|auto] - use at most <n> parallel python processes for test execution, if auto, set to number of available cpus (default: 1) + MAX_VPP_CPUS=[<n>|auto]- use at most <n> cpus for running vpp main and worker threads, if auto, set to number of available cpus (default: auto) + CACHE_OUTPUT=[0|n|no] - disable cache VPP stdout/stderr and log as one block after test finishes (default: yes) + FAILFAST=[1|y|yes] - fail fast if 1, otherwise complete all tests + TIMEOUT=<timeout> - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600) + RETRIES=<n> - retry failed tests <n> times + DEBUG=<type> - set VPP debugging kind + DEBUG=core - detect coredump and load it in gdb on crash + DEBUG=gdb - allow easy debugging by printing VPP PID + and waiting for user input before running + and tearing down a testcase + DEBUG=gdbserver - run gdb inside a gdb server, otherwise + same as above + DEBUG=attach - attach test case to already running vpp in gdb (see test-start-vpp-in-gdb) + STEP=[1|y|yes] - enable stepping through a testcase (for testcase debugging) + SANITY=[0|n|no] - disable sanity import of vpp-api/sanity vpp run before running tests + EXTENDED_TESTS=[1|y|yes] - run extended tests + TEST=<filter> - filter the set of tests: + by file-name - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py + by file-suffix - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py + by wildcard - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*' + e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name + TEST='bfd.*.*' is equivalent to above example of filter by file-suffix + TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class + TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase + TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes + VARIANT=<variant> - specify which march node variant to unit test + e.g. VARIANT=skx test the skx march variants + e.g. VARIANT=icl test the icl march variants + COREDUMP_SIZE=<size> - pass <size> as unix { coredump-size <size> } argument to vpp + e.g. COREDUMP_SIZE=4g + COREDUMP_SIZE=unlimited + COREDUMP_COMPRESS=[1|y|yes] - compress core files if not debugging them + EXTERN_TESTS=<path> - path to out-of-tree test_<name>.py files containing test cases + EXTERN_PLUGINS=<path> - path to out-of-tree plugins to be loaded by vpp under test + EXTERN_COV_DIR=<path> - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report + PROFILE=[1|y|yes] - enable profiling of test framework via cProfile module + PROFILE_SORT_BY=opt - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime) + PROFILE_OUTPUT=file - output profiling info to file - use absolute path (default: stdout) + TEST_DEBUG=[1|y|yes] - enable debugging of the test framework itself (expert) + API_FUZZ=[1|y|yes] - enable VPP api fuzz testing + RND_SEED=<seed> - Seed RND with given seed + + Starting VPP in GDB for use with DEBUG=attach: + + test-start-vpp-in-gdb - start VPP in gdb (release) + test-start-vpp-debug-in-gdb - start VPP in gdb (debug) + + Creating test code coverage report: + + test-cov - generate code coverage report for test framework + test-wipe-cov - wipe code coverage report for test framework + + Verifying code-style: + + test-checkstyle - check PEP8 compliance diff --git a/test/Makefile b/test/Makefile index 207333f0a2c..e1238fb728a 100644 --- a/test/Makefile +++ b/test/Makefile @@ -50,21 +50,10 @@ PROFILE_SORT_BY=cumtime endif ifeq ($(PROFILE),1) -PYTHON_PROFILE_OPTS=-m cProfile $(PROFILE_OUTPUT_OPTS) -s $(PROFILE_SORT_BY) +PYTHON_OPTS="-m cProfile $(PROFILE_OUTPUT_OPTS) -s $(PROFILE_SORT_BY)" FORCE_FOREGROUND=1 endif -UNITTEST_EXTRA_OPTS= -UNITTEST_FAILFAST_OPTS= - -ifeq ($(FAILFAST),1) -UNITTEST_EXTRA_OPTS=-f -endif - -ifneq ($(EXTERN_TESTS),) -UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS) -endif - VENV_PATH=$(TEST_DIR)/venv ifeq ($(TEST_DEBUG),1) @@ -79,6 +68,10 @@ else PYTHON_INTERP=$(PYTHON) endif +ifeq ($(V),) +V=0 +endif + PYTHON_VERSION=$(shell $(PYTHON_INTERP) -c 'import sys; print(sys.version_info.major)') PIP_VERSION=22.0.3 # Keep in sync with requirements.txt @@ -150,32 +143,119 @@ else PLUGIN_SRC_DIR=$(INTERN_PLUGIN_SRC_DIR) endif -define retest-func -@env VPP_IN_GDB=$(VPP_IN_GDB) FORCE_FOREGROUND=$(FORCE_FOREGROUND) FAILED_DIR=$(FAILED_DIR) VENV_PATH=$(VENV_PATH) scripts/setsid_wrapper.sh $(FORCE_FOREGROUND) $(VENV_PATH)/bin/activate $(PYTHON_INTERP) $(PYTHON_PROFILE_OPTS) run_tests.py -d $(TEST_DIR) $(UNITTEST_EXTRA_OPTS) || env FAILED_DIR=$(FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh -endef - .PHONY: sanity -ifeq ($(SANITY),no) -SANITY_IMPORT_VPP_PAPI_CMD=true -SANITY_RUN_VPP_CMD=true -else -SANITY_IMPORT_VPP_PAPI_CMD=source $(VENV_PATH)/bin/activate && python3 sanity_import_vpp_papi.py -SANITY_RUN_VPP_CMD=source $(VENV_PATH)/bin/activate && python3 sanity_run_vpp.py -endif - ifndef TEST_JOBS PARALLEL_ILLEGAL=0 +TEST_JOBS=1 else ifeq ($(FORCE_FOREGROUND),0) PARALLEL_ILLEGAL=0 -else ifeq ($(TEST_JOBS),auto) -PARALLEL_ILLEGAL=0 -else ifeq ($(TEST_JOBS),1) +else ifneq ($(findstring $(TEST_JOBS),1 auto),) PARALLEL_ILLEGAL=0 else PARALLEL_ILLEGAL=1 endif +ifneq ($(DEBUG),) +SANITY=no +endif + +ifneq ($(findstring $(SANITY),0 n no),) +SANITY_IMPORT_VPP_PAPI_CMD=true +ARG0= +else +SANITY_IMPORT_VPP_PAPI_CMD=source $(VENV_PATH)/bin/activate && $(PYTHON_INTERP) sanity_import_vpp_papi.py +ARG0=--sanity +endif + +ARG1= +ifneq ($(findstring $(FAILFAST),1 y yes),) +ARG1=--failfast +endif + +ARG2= +ifneq ($(findstring $(EXTENDED_TESTS),1 y yes),) +ARG2=--extended +endif + +ARG3= +ifneq ($(EXTERN_TESTS),) +ARG3=--test-src-dir $(EXTERN_TESTS) +endif + +ARG4= +ifneq ($(findstring $(FORCE_FOREGROUND),1 y yes),) +ARG4=--force-foreground +endif + +ARG5= +ifneq ($(findstring $(COREDUMP_COMPRESS),1 y yes),) +ARG5=--compress-core +endif + +ARG6= +ifneq ($(findstring $(STEP),1 y yes),) +ARG6=--step +endif + +ARG7= +ifneq ($(findstring $(TESTS_GCOV),1 y yes),) +ARG7=--gcov +endif + +ARG8= +ifneq ($(EXTERN_PLUGINS),) +ARG8=--extern-plugin-dir=$(EXTERN_PLUGINS) +endif + +ARG9= +ifneq ($(DEBUG),) +ARG9=--debug=$(DEBUG) +endif + +ARG10= +ifneq ($(COREDUMP_SIZE),) +ARG10=--coredump-size=$(COREDUMP_SIZE) +endif + +ARG11= +ifneq ($(VARIANT),) +ARG11=--variant=$(VARIANT) +endif + +ARG12=--cache-vpp-output +ifneq ($(findstring $(CACHE_OUTPUT),0 n no),) +ARG12= +endif + +ARG13= +ifneq ($(MAX_VPP_CPUS),) +ARG13=--max-vpp-cpus=$(MAX_VPP_CPUS) +endif + +ARG14= +ifneq ($(TIMEOUT),) +ARG14=--timeout=$(TIMEOUT) +endif + +ARG15= +ifneq ($(findstring $(TEST_DEBUG),1 y yes),) +ARG15=--debug-framework +endif + +ARG16= +ifneq ($(findstring $(API_FUZZ),1 y yes),) +ARG16=--api-fuzz=on +endif + +EXTRA_ARGS=$(ARG0) $(ARG1) $(ARG2) $(ARG3) $(ARG4) $(ARG5) $(ARG6) $(ARG7) $(ARG8) $(ARG9) $(ARG10) $(ARG11) $(ARG12) $(ARG13) $(ARG14) $(ARG15) $(ARG16) + +RUN_TESTS_ARGS=--failed-dir=$(FAILED_DIR) --verbose=$(V) --jobs=$(TEST_JOBS) --filter=$(TEST) --retries=$(RETRIES) --venv-dir=$(VENV_PATH) --vpp-ws-dir=$(WS_ROOT) --vpp-tag=$(TAG) --rnd-seed=$(RND_SEED) --vpp-worker-count="$(VPP_WORKER_COUNT)" --keep-pcaps --python-opts=$(PYTHON_OPTS) $(PLUGIN_PATH_ARGS) $(TEST_PLUGIN_PATH_ARGS) $(EXTRA_ARGS) + +define retest-func +@scripts/run.sh $(RUN_TESTS_ARGS) || env FAILED_DIR=$(FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh +endef + sanity: test-dep @bash -c "test $(PARALLEL_ILLEGAL) -eq 0 ||\ (echo \"*******************************************************************\" &&\ @@ -190,11 +270,6 @@ sanity: test-dep echo \"* 2. execute debugger: gdb python -ex 'run sanity_import_vpp_papi.py'\" &&\ echo \"*******************************************************************\" &&\ false)" - @bash -c "$(SANITY_RUN_VPP_CMD) ||\ - (echo \"*******************************************************************\" &&\ - echo \"* Sanity check failed, cannot run vpp\" &&\ - echo \"*******************************************************************\" &&\ - false)" $(FAILED_DIR): reset @mkdir -p $@ @@ -218,13 +293,12 @@ shell: test-dep echo PYTHONPATH=$(PYTHONPATH);\ echo RND_SEED=$(RND_SEED);\ echo VPP_BUILD_DIR=$(VPP_BUILD_DIR);\ - echo VPP_BIN=$(VPP_BIN);\ echo VPP_PLUGIN_PATH=$(VPP_PLUGIN_PATH);\ echo VPP_TEST_PLUGIN_PATH=$(VPP_TEST_PLUGIN_PATH);\ echo VPP_INSTALL_PATH=$(VPP_INSTALL_PATH);\ echo EXTERN_TESTS=$(EXTERN_TESTS);\ echo EXTERN_PLUGINS=$(EXTERN_PLUGINS);\ - echo EXTERN_COV_DIR=$(EXTERN_COV_DIR);\ + echo EXTERN_COV_DIR=$(EXTERN_COV_DIR);\ echo LD_LIBRARY_PATH=$(LD_LIBRARY_PATH);\ echo '***';\ exec </dev/tty" | bash -i @@ -286,9 +360,7 @@ checkstyle-diff: $(PIP_INSTALL_DONE) .PHONY: start-gdb start-gdb: sanity - $(eval VPP_IN_GDB=1) - $(eval FORCE_FOREGROUND=1) - $(call retest-func) + @bash -c "source $(VENV_PATH)/bin/activate && python3 -c 'from debug import start_vpp_in_gdb; start_vpp_in_gdb()' $(RUN_TESTS_ARGS)" .PHONY: checkstyle checkstyle: $(PIP_INSTALL_DONE) @@ -328,12 +400,13 @@ help: @echo " test-refresh-deps - refresh the Python dependencies for the tests" @echo "" @echo "Arguments controlling test runs:" + @echo "" @echo " V=[0|1|2] - set test verbosity level" @echo " 0=ERROR, 1=INFO, 2=DEBUG" @echo " TEST_JOBS=[<n>|auto] - use at most <n> parallel python processes for test execution, if auto, set to number of available cpus (default: 1)" @echo " MAX_VPP_CPUS=[<n>|auto]- use at most <n> cpus for running vpp main and worker threads, if auto, set to number of available cpus (default: auto)" - @echo " CACHE_OUTPUT=[0|1] - cache VPP stdout/stderr and log as one block after test finishes (default: 1)" - @echo " FAILFAST=[0|1] - fail fast if 1, complete all tests if 0" + @echo " CACHE_OUTPUT=[0|n|no] - disable cache VPP stdout/stderr and log as one block after test finishes (default: yes)" + @echo " FAILFAST=[1|y|yes] - fail fast if 1, otherwise complete all tests" @echo " TIMEOUT=<timeout> - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)" @echo " RETRIES=<n> - retry failed tests <n> times" @echo " DEBUG=<type> - set VPP debugging kind" @@ -344,10 +417,9 @@ help: @echo " DEBUG=gdbserver - run gdb inside a gdb server, otherwise" @echo " same as above" @echo " DEBUG=attach - attach test case to already running vpp in gdb (see test-start-vpp-in-gdb)" - @echo "" - @echo " STEP=[yes|no] - ease debugging by stepping through a testcase" - @echo " SANITY=[yes|no] - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)" - @echo " EXTENDED_TESTS=[1|y] - used by '[re]test-all' & '[re]test-all-debug' to run extended tests" + @echo " STEP=[1|y|yes] - enable stepping through a testcase (for testcase debugging)" + @echo " SANITY=[0|n|no] - disable sanity import of vpp-api/sanity vpp run before running tests" + @echo " EXTENDED_TESTS=[1|y|yes] - run extended tests" @echo " TEST=<filter> - filter the set of tests:" @echo " by file-name - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py" @echo " by file-suffix - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py" @@ -357,44 +429,34 @@ help: @echo " TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class" @echo " TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase" @echo " TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes" - @echo "" @echo " VARIANT=<variant> - specify which march node variant to unit test" @echo " e.g. VARIANT=skx test the skx march variants" @echo " e.g. VARIANT=icl test the icl march variants" - @echo "" @echo " COREDUMP_SIZE=<size> - pass <size> as unix { coredump-size <size> } argument to vpp" @echo " e.g. COREDUMP_SIZE=4g" @echo " COREDUMP_SIZE=unlimited" - @echo " COREDUMP_COMPRESS=1 - compress core files if not debugging them" + @echo " COREDUMP_COMPRESS=[1|y|yes] - compress core files if not debugging them" @echo " EXTERN_TESTS=<path> - path to out-of-tree test_<name>.py files containing test cases" @echo " EXTERN_PLUGINS=<path> - path to out-of-tree plugins to be loaded by vpp under test" @echo " EXTERN_COV_DIR=<path> - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report" - @echo "" - @echo " PROFILE=1 - enable profiling of test framework via cProfile module" + @echo " PROFILE=[1|y|yes] - enable profiling of test framework via cProfile module" @echo " PROFILE_SORT_BY=opt - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)" @echo " PROFILE_OUTPUT=file - output profiling info to file - use absolute path (default: stdout)" - @echo "" - @echo " TEST_DEBUG=1 - turn on debugging of the test framework itself (expert)" - @echo "" - @echo " SKIP_AARCH64=1 - skip tests that are failing on the ARM platorm in FD.io CI" - @echo "" - @echo " RND_SEED=seed - Seed RND with given seed" + @echo " TEST_DEBUG=[1|y|yes] - enable debugging of the test framework itself (expert)" + @echo " API_FUZZ=[1|y|yes] - enable VPP api fuzz testing" + @echo " RND_SEED=<seed> - Seed RND with given seed" @echo "" @echo "Starting VPP in GDB for use with DEBUG=attach:" @echo "" @echo " test-start-vpp-in-gdb - start VPP in gdb (release)" @echo " test-start-vpp-debug-in-gdb - start VPP in gdb (debug)" @echo "" - @echo "Arguments controlling VPP in GDB runs:" - @echo " " - @echo " VPP_IN_GDB_TMP_DIR - specify directory to run VPP IN (default: /tmp/unittest-attach-gdb)" - @echo " VPP_IN_GDB_NO_RMDIR=0 - don't remove existing tmp dir but fail instead" - @echo " VPP_IN_GDB_CMDLINE=1 - add 'interactive' to VPP arguments to run with command line" + @echo "Creating test code coverage report:" @echo "" - @echo "Creating test code coverage report" @echo " test-cov - generate code coverage report for test framework" @echo " test-wipe-cov - wipe code coverage report for test framework" @echo "" - @echo "Verifying code-style" + @echo "Verifying code-style:" + @echo "" @echo " test-checkstyle - check PEP8 compliance" @echo "" diff --git a/test/config.py b/test/config.py new file mode 100644 index 00000000000..b91973f7f95 --- /dev/null +++ b/test/config.py @@ -0,0 +1,290 @@ +import argparse +import os +import psutil +import textwrap +import time + + +def positive_int_or_default(default): + def positive_integer(v): + if v is None or v == "": + return default + return int(v) + return positive_integer + + +def positive_int_or_auto(v): + if v is None or v in ("", "auto"): + return "auto" + if int(v) <= 0: + raise ValueError("value must be positive or auto") + return int(v) + + +def int_or_auto(v): + if v is None or v in ("", "auto"): + return "auto" + if int(v) < 0: + raise ValueError("value must be positive or auto") + return int(v) + + +def int_choice_or_default(options, default): + assert default in options + + def choice(v): + if v is None or v == "": + return default + if int(v) in options: + return int(v) + raise ValueError("invalid choice") + return choice + + +def worker_config(v): + if v is None or v == "": + return 0 + if v.startswith("workers "): + return(int(v.split(" ")[1])) + return int(v) + + +def directory(v): + if not os.path.isdir(v): + raise ValueError(f"provided path '{v}' doesn't exist " + "or is not a directory") + return v + + +def directory_verify_or_create(v): + if not os.path.isdir(v): + os.mkdir(v) + return v + + +parser = argparse.ArgumentParser(description="VPP unit tests", + formatter_class=argparse.RawTextHelpFormatter) + +parser.add_argument("--failfast", action="store_true", + help="stop running tests on first failure") + +parser.add_argument("--test-src-dir", action="append", type=directory, + help="directory containing test files " + "(may be specified multiple times) " + "(VPP_WS_DIR/test is added automatically to the set)") + +default_verbose = 0 + +parser.add_argument("--verbose", action="store", default=default_verbose, + type=int_choice_or_default((0, 1, 2), default_verbose), + help="verbosity setting - 0 - least verbose, " + "2 - most verbose (default: 0)") + +default_test_run_timeout = 600 + +parser.add_argument("--timeout", action="store", + type=positive_int_or_default(default_test_run_timeout), + default=default_test_run_timeout, + metavar="TEST_RUN_TIMEOUT", + help="test run timeout in seconds - per test " + f"(default: {default_test_run_timeout})") + +parser.add_argument("--failed-dir", action="store", type=directory, + help="directory containing failed tests") + +filter_help_string = """\ +expression consists of 3 string selectors separated by '.' separators: + + <file>.<class>.<function> + +- selectors restrict which files/classes/functions are run +- selector can be replaced with '*' or omitted entirely if not needed +- <file> selector is automatically prepended with 'test_' if required +- '.' separators are required only if selector(s) follow(s) + +examples: + +1. all of the following expressions are equivalent and will select + all test classes and functions from test_bfd.py: + 'test_bfd' 'bfd' 'test_bfd..' 'bfd.' 'bfd.*.*' 'test_bfd.*.*' +2. 'bfd.BFDAPITestCase' selects all tests from test_bfd.py, + which are part of BFDAPITestCase class +3. 'bfd.BFDAPITestCase.test_add_bfd' selects a single test named + test_add_bfd from test_bfd.py/BFDAPITestCase +4. '.*.test_add_bfd' selects all test functions named test_add_bfd + from all files/classes +""" +parser.add_argument("--filter", action="store", + metavar="FILTER_EXPRESSION", help=filter_help_string) + +default_retries = 0 + +parser.add_argument("--retries", action="store", default=default_retries, + type=positive_int_or_default(default_retries), + help="retry failed tests RETRIES times") + +parser.add_argument("--step", action="store_true", default=False, + help="enable stepping through tests") + +debug_help_string = """\ +attach - attach to already running vpp +core - detect coredump and load core in gdb on crash +gdb - print VPP PID and pause allowing attaching gdb +gdbserver - same as above, but run gdb in gdbserver +""" + +parser.add_argument("--debug", action="store", + choices=["attach", "core", "gdb", "gdbserver"], + help=debug_help_string) + +parser.add_argument("--debug-framework", action="store_true", + help="enable internal test framework debugging") + +parser.add_argument("--compress-core", action="store_true", + help="compress core files if not debugging them") + +parser.add_argument("--extended", action="store_true", + help="run extended tests") + +parser.add_argument("--sanity", action="store_true", + help="perform sanity vpp run before running tests") + +parser.add_argument("--force-foreground", action="store_true", + help="force running in foreground - don't fork") + +parser.add_argument("--jobs", action="store", type=positive_int_or_auto, + default="auto", help="maximum concurrent test jobs") + +parser.add_argument("--venv-dir", action="store", + type=directory, help="path to virtual environment") + +default_rnd_seed = time.time() +parser.add_argument("--rnd-seed", action="store", default=default_rnd_seed, + type=positive_int_or_default(default_rnd_seed), + help="random generator seed (default: current time)") + +parser.add_argument("--vpp-worker-count", action="store", type=worker_config, + default=0, help="number of vpp workers") + +parser.add_argument("--gcov", action="store_true", + default=False, help="running gcov tests") + +parser.add_argument("--cache-vpp-output", action="store_true", default=False, + help="cache VPP stdout/stderr and log as one block " + "after test finishes") + +parser.add_argument("--vpp-ws-dir", action="store", required=True, + type=directory, help="vpp workspace directory") + +parser.add_argument("--vpp-tag", action="store", default="vpp_debug", + metavar="VPP_TAG", required=True, + help="vpp tag (e.g. vpp, vpp_debug, vpp_gcov)") + +parser.add_argument("--vpp", action="store", help="path to vpp binary " + "(default: derive from VPP_WS_DIR and VPP_TAG)") + +parser.add_argument("--vpp-install-dir", type=directory, + action="store", help="path to vpp install directory" + "(default: derive from VPP_WS_DIR and VPP_TAG)") + +parser.add_argument("--vpp-build-dir", action="store", type=directory, + help="vpp build directory" + "(default: derive from VPP_WS_DIR and VPP_TAG)") + +parser.add_argument("--vpp-plugin-dir", action="append", type=directory, + help="directory containing vpp plugins" + "(default: derive from VPP_WS_DIR and VPP_TAG)") + +parser.add_argument("--vpp-test-plugin-dir", action="append", type=directory, + help="directory containing vpp api test plugins" + "(default: derive from VPP_WS_DIR and VPP_TAG)") + +parser.add_argument("--extern-plugin-dir", action="append", type=directory, + default=[], help="directory containing external plugins") + +parser.add_argument("--coredump-size", action="store", default="unlimited", + help="specify vpp coredump size") + +parser.add_argument("--max-vpp-cpus", action="store", type=int_or_auto, + default=0, help="max cpus used by vpp") + +variant_help_string = """\ +specify which march node variant to unit test + e.g. --variant=skx - test the skx march variants + e.g. --variant=icl - test the icl march variants +""" + +parser.add_argument("--variant", action="store", help=variant_help_string) + +parser.add_argument("--api-fuzz", action="store", default=None, + help="specify api fuzzing parameters") + +parser.add_argument("--wipe-tmp-dir", action="store_true", default=True, + help="remove test tmp directory before running test") + +parser.add_argument("--tmp-dir", action="store", default="/tmp", + type=directory_verify_or_create, + help="directory where to store test temporary directories") + +parser.add_argument("--log-dir", action="store", + type=directory_verify_or_create, + help="directory where to store directories " + "containing log files (default: --tmp-dir)") + +default_keep_pcaps = False +parser.add_argument("--keep-pcaps", action="store_true", + default=default_keep_pcaps, + help="if set, keep all pcap files from a test run" + f" (default: {default_keep_pcaps})") + +config = parser.parse_args() + +ws = config.vpp_ws_dir +br = f"{ws}/build-root" +tag = config.vpp_tag + +if config.vpp_install_dir is None: + config.vpp_install_dir = f"{br}/install-{tag}-native" + +if config.vpp is None: + config.vpp = f"{config.vpp_install_dir}/vpp/bin/vpp" + +if config.vpp_build_dir is None: + config.vpp_build_dir = f"{br}/build-{tag}-native" + +libs = ["lib", "lib64"] + +if config.vpp_plugin_dir is None: + config.vpp_plugin_dir = [ + f"{config.vpp_install_dir}/vpp/{lib}/vpp_plugins" for lib in libs] + +if config.vpp_test_plugin_dir is None: + config.vpp_test_plugin_dir = [ + f"{config.vpp_install_dir}/vpp/{lib}/vpp_api_test_plugins" + for lib in libs] + +test_dirs = [f"{ws}/test"] + +if config.test_src_dir is not None: + test_dirs.extend(config.test_src_dir) + +config.test_src_dir = test_dirs + + +if config.venv_dir is None: + config.venv_dir = f"{ws}/test/venv" + +available_cpus = psutil.Process().cpu_affinity() +num_cpus = len(available_cpus) + +if config.max_vpp_cpus == 'auto': + max_vpp_cpus = num_cpus +elif config.max_vpp_cpus > 0: + max_vpp_cpus = min(config.max_vpp_cpus, num_cpus) +else: + max_vpp_cpus = num_cpus + +if __name__ == "__main__": + print("Provided arguments:") + for i in config.__dict__: + print(f" {i} is {config.__dict__[i]}") diff --git a/test/cpu_config.py b/test/cpu_config.py deleted file mode 100644 index b4e5d1ab767..00000000000 --- a/test/cpu_config.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -import psutil - -available_cpus = psutil.Process().cpu_affinity() -num_cpus = len(available_cpus) - -max_vpp_cpus = os.getenv("MAX_VPP_CPUS", "auto").lower() - -if max_vpp_cpus == "auto": - max_vpp_cpus = num_cpus -else: - try: - max_vpp_cpus = int(max_vpp_cpus) - except ValueError as e: - raise ValueError("Invalid MAX_VPP_CPUS value specified, valid " - "values are a positive integer or 'auto'") from e - if max_vpp_cpus <= 0: - raise ValueError("Invalid MAX_VPP_CPUS value specified, valid " - "values are a positive integer or 'auto'") - if max_vpp_cpus > num_cpus: - max_vpp_cpus = num_cpus diff --git a/test/debug.py b/test/debug.py index a911a78b47f..0ab143201c8 100644 --- a/test/debug.py +++ b/test/debug.py @@ -6,7 +6,7 @@ import sys from sanity_run_vpp import SanityTestCase from shutil import rmtree -from cpu_config import available_cpus +from config import available_cpus gdb_path = '/usr/bin/gdb' @@ -33,21 +33,12 @@ def start_vpp_in_gdb(): # but any test case class could be used ... SanityTestCase.set_debug_flags("attach") SanityTestCase.tempdir = SanityTestCase.get_tempdir() - if os.path.exists(SanityTestCase.tempdir): - if os.getenv("VPP_IN_GDB_NO_RMDIR", "0") in ["1", "y", "yes"]: - raise FileExistsError( - "Temporary directory exists and removal denied.") - print("Removing existing temp dir '%s'." % SanityTestCase.tempdir) - rmtree(SanityTestCase.tempdir) - print("Creating temp dir '%s'." % SanityTestCase.tempdir) - os.mkdir(SanityTestCase.tempdir) SanityTestCase.assign_cpus( available_cpus[:SanityTestCase.get_cpus_required()]) SanityTestCase.setUpConstants() vpp_cmdline = SanityTestCase.vpp_cmdline - if os.getenv("VPP_IN_GDB_CMDLINE", "y").lower() in ["1", "y", "yes"]: - print("Hacking cmdline to make VPP interactive.") - vpp_cmdline.insert(vpp_cmdline.index("nodaemon"), "interactive") + print("Hacking cmdline to make VPP interactive.") + vpp_cmdline.insert(vpp_cmdline.index("nodaemon"), "interactive") print("VPP cmdline is %s" % " ".join(vpp_cmdline)) print("Running GDB.") diff --git a/test/discover_tests.py b/test/discover_tests.py index 1e581a57b6f..7f05c3184ff 100755 --- a/test/discover_tests.py +++ b/test/discover_tests.py @@ -41,18 +41,3 @@ def discover_tests(directory, callback, ignore_path): def print_callback(file_name, cls, method): print("%s.%s.%s" % (file_name, cls.__name__, method)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description="Discover VPP unit tests") - parser.add_argument("-d", "--dir", action='append', type=str, - help="directory containing test files " - "(may be specified multiple times)") - args = parser.parse_args() - if args.dir is None: - args.dir = "." - - ignore_path = os.getenv("VENV_PATH", "") - suite = unittest.TestSuite() - for d in args.dir: - discover_tests(d, print_callback, ignore_path) diff --git a/test/framework.py b/test/framework.py index 2c74a03bf37..572207db3f8 100644 --- a/test/framework.py +++ b/test/framework.py @@ -9,13 +9,13 @@ import select import signal import subprocess import unittest -import tempfile +import re import time import faulthandler import random import copy -import psutil import platform +import shutil from collections import deque from threading import Thread, Event from inspect import getdoc, isclass @@ -27,6 +27,7 @@ from struct import pack, unpack import scapy.compat from scapy.packet import Raw, Packet +from config import config, available_cpus, num_cpus, max_vpp_cpus import hook as hookmodule from vpp_pg_interface import VppPGInterface from vpp_sub_interface import VppSubInterface @@ -45,7 +46,6 @@ from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest from scapy.layers.inet6 import ICMPv6EchoReply -from cpu_config import available_cpus, num_cpus, max_vpp_cpus logger = logging.getLogger(__name__) @@ -61,27 +61,7 @@ TEST_RUN = 4 SKIP_CPU_SHORTAGE = 5 -class BoolEnvironmentVariable(object): - - def __init__(self, env_var_name, default='n', true_values=None): - self.name = env_var_name - self.default = default - self.true_values = true_values if true_values is not None else \ - ("y", "yes", "1") - - def __bool__(self): - return os.getenv(self.name, self.default).lower() in self.true_values - - if sys.version_info[0] == 2: - __nonzero__ = __bool__ - - def __repr__(self): - return 'BoolEnvironmentVariable(%r, default=%r, true_values=%r)' % \ - (self.name, self.default, self.true_values) - - -debug_framework = BoolEnvironmentVariable('TEST_DEBUG') -if debug_framework: +if config.debug_framework: import debug_internal """ @@ -175,7 +155,7 @@ def pump_output(testclass): limit = -1 stdout_fragment = split[-1] testclass.vpp_stdout_deque.extend(split[:limit]) - if not testclass.cache_vpp_output: + if not config.cache_vpp_output: for line in split[:limit]: testclass.logger.info( "VPP STDOUT: %s" % line.rstrip("\n")) @@ -193,7 +173,7 @@ def pump_output(testclass): stderr_fragment = split[-1] testclass.vpp_stderr_deque.extend(split[:limit]) - if not testclass.cache_vpp_output: + if not config.cache_vpp_output: for line in split[:limit]: testclass.logger.error( "VPP STDERR: %s" % line.rstrip("\n")) @@ -201,13 +181,6 @@ def pump_output(testclass): # flag will take care of properly terminating the loop -def _is_skip_aarch64_set(): - return BoolEnvironmentVariable('SKIP_AARCH64') - - -is_skip_aarch64_set = _is_skip_aarch64_set() - - def _is_platform_aarch64(): return platform.machine() == 'aarch64' @@ -215,35 +188,6 @@ def _is_platform_aarch64(): is_platform_aarch64 = _is_platform_aarch64() -def _running_extended_tests(): - return BoolEnvironmentVariable("EXTENDED_TESTS") - - -running_extended_tests = _running_extended_tests() - - -def _running_gcov_tests(): - return BoolEnvironmentVariable("GCOV_TESTS") - - -running_gcov_tests = _running_gcov_tests() - - -def get_environ_vpp_worker_count(): - worker_config = os.getenv("VPP_WORKER_CONFIG", None) - if worker_config: - elems = worker_config.split(" ") - if elems[0] != "workers" or len(elems) != 2: - raise ValueError("Wrong VPP_WORKER_CONFIG == '%s' value." % - worker_config) - return int(elems[1]) - else: - return 0 - - -environ_vpp_worker_count = get_environ_vpp_worker_count() - - class KeepAliveReporter(object): """ Singleton object which reports test start to parent process @@ -277,7 +221,7 @@ class KeepAliveReporter(object): else: desc = test.id() - self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid)) + self.pipe.send((desc, config.vpp, test.tempdir, test.vpp.pid)) class TestCaseTag(Enum): @@ -411,7 +355,7 @@ class VppTestCase(CPUInterface, unittest.TestCase): if cls.has_tag(TestCaseTag.FIXME_VPP_WORKERS): cls.vpp_worker_count = 0 else: - cls.vpp_worker_count = environ_vpp_worker_count + cls.vpp_worker_count = config.vpp_worker_count return cls.vpp_worker_count @classmethod @@ -421,42 +365,38 @@ class VppTestCase(CPUInterface, unittest.TestCase): @classmethod def setUpConstants(cls): """ Set-up the test case class based on environment variables """ - cls.step = BoolEnvironmentVariable('STEP') - # inverted case to handle '' == True - c = os.getenv("CACHE_OUTPUT", "1") - cls.cache_vpp_output = False if c.lower() in ("n", "no", "0") else True - cls.vpp_bin = os.getenv('VPP_BIN', "vpp") - extern_plugin_path = os.getenv('EXTERN_PLUGINS') + cls.step = config.step + cls.plugin_path = ":".join(config.vpp_plugin_dir) + cls.test_plugin_path = ":".join(config.vpp_test_plugin_dir) + cls.extern_plugin_path = ":".join(config.extern_plugin_dir) debug_cli = "" if cls.step or cls.debug_gdb or cls.debug_gdbserver: debug_cli = "cli-listen localhost:5002" - coredump_size = None - size = os.getenv("COREDUMP_SIZE") - if size is not None: - coredump_size = "coredump-size %s" % size - if coredump_size is None: + size = re.search(r"\d+[gG]", config.coredump_size) + if size: + coredump_size = f"coredump-size {config.coredump_size}".lower() + else: coredump_size = "coredump-size unlimited" - - default_variant = os.getenv("VARIANT") + default_variant = config.variant if default_variant is not None: default_variant = "defaults { %s 100 }" % default_variant else: default_variant = "" - api_fuzzing = os.getenv("API_FUZZ") + api_fuzzing = config.api_fuzz if api_fuzzing is None: api_fuzzing = 'off' cls.vpp_cmdline = [ - cls.vpp_bin, + config.vpp, "unix", "{", "nodaemon", debug_cli, "full-coredump", coredump_size, "runtime-dir", cls.tempdir, "}", "api-trace", "{", "on", "}", "api-segment", "{", "prefix", cls.get_api_segment_prefix(), "}", "cpu", "{", "main-core", str(cls.cpus[0]), ] - if extern_plugin_path is not None: + if cls.extern_plugin_path not in (None, ""): cls.extra_vpp_plugin_config.append( - "add-path %s" % extern_plugin_path) + "add-path %s" % cls.extern_plugin_path) if cls.get_vpp_worker_count(): cls.vpp_cmdline.extend([ "corelist-workers", ",".join([str(x) for x in cls.cpus[1:]])]) @@ -495,15 +435,14 @@ class VppTestCase(CPUInterface, unittest.TestCase): print(single_line_delim) print("You can debug VPP using:") if cls.debug_gdbserver: - print("sudo gdb " + cls.vpp_bin + - " -ex 'target remote localhost:{port}'" - .format(port=cls.gdbserver_port)) + print(f"sudo gdb {config.vpp} " + f"-ex 'target remote localhost:{cls.gdbserver_port}'") print("Now is the time to attach gdb by running the above " "command, set up breakpoints etc., then resume VPP from " "within gdb by issuing the 'continue' command") cls.gdbserver_port += 1 elif cls.debug_gdb: - print("sudo gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid) + print(f"sudo gdb {config.vpp} -ex 'attach {cls.vpp.pid}'") print("Now is the time to attach gdb by running the above " "command and set up breakpoints etc., then resume VPP from" " within gdb by issuing the 'continue' command") @@ -585,11 +524,23 @@ class VppTestCase(CPUInterface, unittest.TestCase): @classmethod def get_tempdir(cls): - if cls.debug_attach: - return os.getenv("VPP_IN_GDB_TMP_DIR", - "/tmp/unittest-attach-gdb") - else: - return tempfile.mkdtemp(prefix='vpp-unittest-%s-' % cls.__name__) + tmpdir = f"{config.tmp_dir}/vpp-unittest-{cls.__name__}" + if config.wipe_tmp_dir: + shutil.rmtree(tmpdir, ignore_errors=True) + os.mkdir(tmpdir) + return tmpdir + + @classmethod + def create_file_handler(cls): + if config.log_dir is None: + cls.file_handler = FileHandler(f"{cls.tempdir}/log.txt") + return + + logdir = f"{config.log_dir}/vpp-unittest-{cls.__name__}" + if config.wipe_tmp_dir: + shutil.rmtree(logdir, ignore_errors=True) + os.mkdir(logdir) + cls.file_handler = FileHandler(f"{logdir}/log.txt") @classmethod def setUpClass(cls): @@ -599,15 +550,13 @@ class VppTestCase(CPUInterface, unittest.TestCase): """ super(VppTestCase, cls).setUpClass() cls.logger = get_logger(cls.__name__) - seed = os.environ["RND_SEED"] - random.seed(seed) + random.seed(config.rnd_seed) if hasattr(cls, 'parallel_handler'): cls.logger.addHandler(cls.parallel_handler) cls.logger.propagate = False - d = os.getenv("DEBUG", None) - cls.set_debug_flags(d) + cls.set_debug_flags(config.debug) cls.tempdir = cls.get_tempdir() - cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir) + cls.create_file_handler() cls.file_handler.setFormatter( Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s', datefmt="%H:%M:%S")) @@ -617,7 +566,7 @@ class VppTestCase(CPUInterface, unittest.TestCase): os.chdir(cls.tempdir) cls.logger.info("Temporary dir is %s, api socket is %s", cls.tempdir, cls.get_api_sock_path()) - cls.logger.debug("Random seed is %s", seed) + cls.logger.debug("Random seed is %s", config.rnd_seed) cls.setUpConstants() cls.reset_packet_infos() cls._pcaps = [] @@ -636,7 +585,7 @@ class VppTestCase(CPUInterface, unittest.TestCase): cls.run_vpp() cls.reporter.send_keep_alive(cls, 'setUpClass') VppTestResult.current_test_case_info = TestCaseInfo( - cls.logger, cls.tempdir, cls.vpp.pid, cls.vpp_bin) + cls.logger, cls.tempdir, cls.vpp.pid, config.vpp) cls.vpp_stdout_deque = deque() cls.vpp_stderr_deque = deque() if not cls.debug_attach: @@ -785,7 +734,7 @@ class VppTestCase(CPUInterface, unittest.TestCase): cls.quit() cls.file_handler.close() cls.reset_packet_infos() - if debug_framework: + if config.debug_framework: debug_internal.on_tear_down_class(cls) def show_commands_at_teardown(self): @@ -890,7 +839,7 @@ class VppTestCase(CPUInterface, unittest.TestCase): def pg_start(cls, trace=True): """ Enable the PG, wait till it is done, then clean up """ for (intf, worker) in cls._old_pcaps: - intf.rename_old_pcap_file(intf.get_in_path(worker), + intf.handle_old_pcap_file(intf.get_in_path(worker), intf.in_history_counter) cls._old_pcaps = [] if trace: @@ -1439,7 +1388,7 @@ class VppTestResult(unittest.TestResult): def symlink_failed(self): if self.current_test_case_info: try: - failed_dir = os.getenv('FAILED_DIR') + failed_dir = config.failed_dir link_path = os.path.join( failed_dir, '%s-FAILED' % diff --git a/test/log.py b/test/log.py index 5fd91c794e8..9701aecc04a 100644 --- a/test/log.py +++ b/test/log.py @@ -4,6 +4,8 @@ import sys import os import logging +from config import config + """ @var formatting delimiter consisting of '=' characters """ double_line_delim = '=' * 78 """ @var formatting delimiter consisting of '-' characters """ @@ -24,15 +26,12 @@ class ColorFormatter(logging.Formatter): if hasattr(record, 'color'): message = colorize(message, record.color) return message -try: - verbose = int(os.getenv("V", 0)) -except: - verbose = 0 + # 40 = ERROR, 30 = WARNING, 20 = INFO, 10 = DEBUG, 0 = NOTSET (all messages) -if verbose >= 2: +if config.verbose >= 2: log_level = 10 -elif verbose == 1: +elif config.verbose == 1: log_level = 20 else: log_level = 40 @@ -66,6 +65,7 @@ def get_parallel_logger(stream): logger.addHandler(handler) return logger + # Static variables to store color formatting strings. # # These variables (RED, GREEN, YELLOW and LPURPLE) are used to configure diff --git a/test/run_tests.py b/test/run_tests.py index 5d091ad253f..1d194ad96ca 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -8,22 +8,24 @@ import unittest import argparse import time import threading +import traceback import signal import re from multiprocessing import Process, Pipe, get_context from multiprocessing.queues import Queue from multiprocessing.managers import BaseManager import framework -from framework import VppTestRunner, running_extended_tests, VppTestCase, \ +from config import config, num_cpus, available_cpus, max_vpp_cpus +from framework import VppTestRunner, VppTestCase, \ get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \ TEST_RUN, SKIP_CPU_SHORTAGE from debug import spawn_gdb, start_vpp_in_gdb from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \ colorize, single_line_delim from discover_tests import discover_tests +import sanity_run_vpp from subprocess import check_output, CalledProcessError from util import check_core_path, get_core_path, is_core_present -from cpu_config import num_cpus, max_vpp_cpus, available_cpus # timeout which controls how long the child has to finish after seeing # a core dump in test temporary directory. If this is exceeded, parent assumes @@ -126,9 +128,9 @@ def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue, VppTestCase.parallel_handler = logger.handlers[0] result = VppTestRunner(keep_alive_pipe=keep_alive_pipe, descriptions=descriptions, - verbosity=verbose, + verbosity=config.verbose, result_pipe=result_pipe, - failfast=failfast, + failfast=config.failfast, print_summary=False).run(suite) finished_pipe.send(result.wasSuccessful()) finished_pipe.close() @@ -241,8 +243,7 @@ def handle_failed_suite(logger, last_test_temp_dir, vpp_pid): if last_test_temp_dir: # Need to create link in case of a timeout or core dump without failure lttd = os.path.basename(last_test_temp_dir) - failed_dir = os.getenv('FAILED_DIR') - link_path = '%s%s-FAILED' % (failed_dir, lttd) + link_path = '%s%s-FAILED' % (config.failed_dir, lttd) if not os.path.exists(link_path): os.symlink(last_test_temp_dir, link_path) logger.error("Symlink to failed testcase directory: %s -> %s" @@ -272,8 +273,7 @@ def handle_failed_suite(logger, last_test_temp_dir, vpp_pid): except Exception as e: logger.exception("Unexpected error running `file' utility " "on core-file") - logger.error("gdb %s %s" % - (os.getenv('VPP_BIN', 'vpp'), core_path)) + logger.error(f"gdb {config.vpp_bin} {core_path}") if vpp_pid: # Copy api post mortem @@ -292,7 +292,7 @@ def check_and_handle_core(vpp_binary, tempdir, core_crash_test): print(single_line_delim) spawn_gdb(vpp_binary, get_core_path(tempdir)) print(single_line_delim) - elif compress_core: + elif config.compress_core: print("Compressing core-file in test directory `%s'" % tempdir) os.system("gzip %s" % get_core_path(tempdir)) @@ -312,7 +312,7 @@ def process_finished_testsuite(wrapped_testcase_suite, results.append(wrapped_testcase_suite.result) finished_testcase_suites.add(wrapped_testcase_suite) stop_run = False - if failfast and not wrapped_testcase_suite.was_successful(): + if config.failfast and not wrapped_testcase_suite.was_successful(): stop_run = True if not wrapped_testcase_suite.was_successful(): @@ -421,7 +421,7 @@ def run_forked(testcase_suites): continue fail = False - if wrapped_testcase_suite.last_heard + test_timeout < \ + if wrapped_testcase_suite.last_heard + config.timeout < \ time.time(): fail = True wrapped_testcase_suite.logger.critical( @@ -515,7 +515,7 @@ def run_forked(testcase_suites): raise finally: read_from_testcases.clear() - stdouterr_thread.join(test_timeout) + stdouterr_thread.join(config.timeout) manager.shutdown() handle_cores(failed_wrapped_testcases) @@ -566,11 +566,8 @@ class SplitToSuitesCallback: self.filtered.addTest(test_method) -test_option = "TEST" - - -def parse_test_option(): - f = os.getenv(test_option, None) +def parse_test_filter(test_filter): + f = test_filter filter_file_name = None filter_class_name = None filter_func_name = None @@ -807,46 +804,32 @@ def parse_results(results): return return_code, results_per_suite.rerun -def parse_digit_env(env_var, default): - value = os.getenv(env_var, default) - if value != default: - if value.isdigit(): - value = int(value) - else: - print('WARNING: unsupported value "%s" for env var "%s",' - 'defaulting to %s' % (value, env_var, default)) - value = default - return value - - if __name__ == '__main__': - verbose = parse_digit_env("V", 0) + print(f"Config is: {config}") - test_timeout = parse_digit_env("TIMEOUT", 600) # default = 10 minutes + if config.sanity: + print("Running sanity test case.") + try: + rc = sanity_run_vpp.main() + if rc != 0: + sys.exit(rc) + except Exception as e: + print(traceback.format_exc()) + print("Couldn't run sanity test case.") + sys.exit(-1) test_finished_join_timeout = 15 - retries = parse_digit_env("RETRIES", 0) + debug_gdb = config.debug in ["gdb", "gdbserver", "attach"] + debug_core = config.debug == "core" - debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver", "attach"] - - debug_core = os.getenv("DEBUG", "").lower() == "core" - compress_core = framework.BoolEnvironmentVariable("CORE_COMPRESS") - - if os.getenv("VPP_IN_GDB", "n").lower() in ["1", "y", "yes"]: - start_vpp_in_gdb() - exit() - - step = framework.BoolEnvironmentVariable("STEP") - force_foreground = framework.BoolEnvironmentVariable("FORCE_FOREGROUND") - - run_interactive = debug or step or force_foreground + run_interactive = debug_gdb or config.step or config.force_foreground max_concurrent_tests = 0 print(f"OS reports {num_cpus} available cpu(s).") - test_jobs = os.getenv("TEST_JOBS", "1").lower() # default = 1 process + test_jobs = config.jobs if test_jobs == 'auto': if run_interactive: max_concurrent_tests = 1 @@ -856,15 +839,7 @@ if __name__ == '__main__': print(f"Running at most {max_concurrent_tests} python test " "processes concurrently.") else: - try: - test_jobs = int(test_jobs) - except ValueError as e: - raise ValueError("Invalid TEST_JOBS value specified, valid " - "values are a positive integer or 'auto'") from e - if test_jobs <= 0: - raise ValueError("Invalid TEST_JOBS value specified, valid " - "values are a positive integer or 'auto'") - max_concurrent_tests = int(test_jobs) + max_concurrent_tests = test_jobs print(f"Running at most {max_concurrent_tests} python test processes " "concurrently as set by 'TEST_JOBS'.") @@ -876,27 +851,20 @@ if __name__ == '__main__': 'STEP is set) in parallel (TEST_JOBS is more than 1) is not ' 'supported') - parser = argparse.ArgumentParser(description="VPP unit tests") - parser.add_argument("-f", "--failfast", action='store_true', - help="fast failure flag") - parser.add_argument("-d", "--dir", action='append', type=str, - help="directory containing test files " - "(may be specified multiple times)") - args = parser.parse_args() - failfast = args.failfast descriptions = True print("Running tests using custom test runner.") - filter_file, filter_class, filter_func = parse_test_option() + filter_file, filter_class, filter_func = \ + parse_test_filter(config.filter) - print("Active filters: file=%s, class=%s, function=%s" % ( + print("Selected filters: file=%s, class=%s, function=%s" % ( filter_file, filter_class, filter_func)) filter_cb = FilterByTestOption(filter_file, filter_class, filter_func) - ignore_path = os.getenv("VENV_PATH", None) + ignore_path = config.venv_dir cb = SplitToSuitesCallback(filter_cb) - for d in args.dir: + for d in config.test_src_dir: print("Adding tests from directory tree %s" % d) discover_tests(d, cb, ignore_path) @@ -925,10 +893,10 @@ if __name__ == '__main__': print("%s out of %s tests match specified filters" % ( tests_amount, tests_amount + cb.filtered.countTestCases())) - if not running_extended_tests: + if not config.extended: print("Not running extended tests (some tests will be skipped)") - attempts = retries + 1 + attempts = config.retries + 1 if attempts > 1: print("Perform %s attempts to pass the suite..." % attempts) @@ -945,8 +913,8 @@ if __name__ == '__main__': suite.assign_cpus([]) cpu_shortage = True full_suite.addTests(suites) - result = VppTestRunner(verbosity=verbose, - failfast=failfast, + result = VppTestRunner(verbosity=config.verbose, + failfast=config.failfast, print_summary=True).run(full_suite) was_successful = result.wasSuccessful() if not was_successful: diff --git a/test/sanity_run_vpp.py b/test/sanity_run_vpp.py index 4b21de12dfe..b923c791b61 100644 --- a/test/sanity_run_vpp.py +++ b/test/sanity_run_vpp.py @@ -24,7 +24,7 @@ class SanityTestCase(VppTestCase): pass -if __name__ == '__main__': +def main(): rc = 0 tc = SanityTestCase x, y = Pipe() @@ -46,5 +46,8 @@ if __name__ == '__main__': print('Sanity test case passed.') else: print('Sanity test case failed.') + return rc + - sys.exit(rc) +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/scripts/run.sh b/test/scripts/run.sh new file mode 100755 index 00000000000..544a536ed36 --- /dev/null +++ b/test/scripts/run.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +ff="0" +items= +for i in "$@" +do +case $i in + --venv-dir=*) + venv_dir="${i#*=}" + if [ -d $venv_dir ] + then + venv_dir=$(cd $venv_dir; pwd) + else + echo "ERROR: '$venv_dir' is not a directory" + exit 1 + fi + items="$items --venv-dir=\"$venv_dir\"" + ;; + --vpp-ws-dir=*) + ws_dir="${i#*=}" + if [ -d $ws_dir ] + then + ws_dir=$(cd $ws_dir; pwd) + else + echo "ERROR: '$ws_dir' is not a directory" + exit 1 + fi + items="$items --vpp-ws-dir=\"$ws_dir\"" + ;; + --force-foreground) + ff="1" + items="$items \"$i\"" + ;; + --vpp-tag=*) + tag="${i#*=}" + items="$items \"$i\"" + ;; + --python-opts=*) + python_opts="${i#*=}" + ;; + *) + # unknown option - skip + items="$items \"$i\"" + ;; +esac +done + +extra_args="" +if [ -z "$ws_dir" ] +then + ws_dir=$(pwd) + echo "Argument --vpp-ws-dir not specified, defaulting to '$ws_dir'" + extra_args="$extra_args --vpp-ws-dir=$ws_dir" +fi + +if [ -z "$venv_dir" ] +then + venv_dir="$ws_dir/test/venv" + echo "Argument --venv-path not specified, defaulting to '$venv_dir'" + extra_args="$extra_args --venv-dir=$venv_dir" +fi + +if [ -z "$tag" ] +then + tag="vpp_debug" + echo "Argument --vpp-tag not specified, defaulting to '$tag'" + extra_args="$extra_args --vpp-tag=$tag" +fi + +eval set -- $items +$ws_dir/test/scripts/setsid_wrapper.sh $ws_dir/test/scripts/run_in_venv_with_cleanup.sh $ff $venv_dir/bin/activate python3 $python_opts $ws_dir/test/run_tests.py $extra_args $* diff --git a/test/scripts/setsid_wrapper.sh b/test/scripts/setsid_wrapper.sh index 6d63426becc..030c3b079de 100755 --- a/test/scripts/setsid_wrapper.sh +++ b/test/scripts/setsid_wrapper.sh @@ -1,10 +1,15 @@ #!/bin/bash -if [[ "$1" == "1" ]] +cmd=$1 +force_foreground=$2 +shift +shift + +if [[ "$force_foreground" == "1" ]] then - setsid scripts/run_in_venv_with_cleanup.sh $* + setsid $cmd $force_foreground $* else - setsid scripts/run_in_venv_with_cleanup.sh $* & + setsid $cmd $force_foreground $* & pid=$! trap "echo setsid_wrapper.sh: got signal, killing child pid ${pid}; kill ${pid}; sleep .1;" SIGINT SIGTERM wait ${pid} diff --git a/test/test_acl_plugin_conns.py b/test/test_acl_plugin_conns.py index c7941fa150b..cbf0ab37cf8 100644 --- a/test/test_acl_plugin_conns.py +++ b/test/test_acl_plugin_conns.py @@ -2,7 +2,8 @@ """ ACL plugin extended stateful tests """ import unittest -from framework import VppTestCase, VppTestRunner, running_extended_tests +from config import config +from framework import VppTestCase, VppTestRunner from scapy.layers.l2 import Ether from scapy.packet import Raw from scapy.layers.inet import IP, UDP, TCP @@ -118,7 +119,7 @@ class Conn(L4_Conn): return new_rule -@unittest.skipUnless(running_extended_tests, "part of extended tests") +@unittest.skipUnless(config.extended, "part of extended tests") class ACLPluginConnTestCase(VppTestCase): """ ACL plugin connection-oriented extended testcases """ diff --git a/test/test_acl_plugin_macip.py b/test/test_acl_plugin_macip.py index 5edd7b03258..5353c16d331 100644 --- a/test/test_acl_plugin_macip.py +++ b/test/test_acl_plugin_macip.py @@ -17,7 +17,7 @@ from scapy.layers.l2 import Ether from scapy.layers.inet import IP, UDP from scapy.layers.inet6 import IPv6 -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from vpp_lo_interface import VppLoInterface from vpp_l2 import L2_PORT_TYPE from vpp_sub_interface import L2_VTR_OP, VppSubInterface, VppDot1QSubint, \ diff --git a/test/test_adl.py b/test/test_adl.py index 4a996fc5c90..0463e258013 100644 --- a/test/test_adl.py +++ b/test/test_adl.py @@ -2,7 +2,7 @@ import unittest -from framework import VppTestCase, VppTestRunner, running_gcov_tests +from framework import VppTestCase, VppTestRunner from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath diff --git a/test/test_bfd.py b/test/test_bfd.py index 174db3ed54a..d1c7b92d89d 100644 --- a/test/test_bfd.py +++ b/test/test_bfd.py @@ -20,10 +20,11 @@ from scapy.layers.inet6 import IPv6 from scapy.layers.l2 import Ether, GRE from scapy.packet import Raw +from config import config from bfd import VppBFDAuthKey, BFD, BFDAuthType, VppBFDUDPSession, \ BFDDiagCode, BFDState, BFD_vpp_echo from framework import tag_fixme_vpp_workers -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from framework import tag_run_solo from util import ppp from vpp_ip import DpoProto @@ -1122,7 +1123,7 @@ class BFD4TestCase(VppTestCase): "Poll bit set in BFD packet") # returning inconsistent results requiring retries in per-patch tests - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_poll_response(self): """ test correct response to control frame with poll bit set """ bfd_session_up(self) @@ -1950,7 +1951,7 @@ class BFDFIBTestCase(VppTestCase): packet[IPv6].dst) -@unittest.skipUnless(running_extended_tests, "part of extended tests") +@unittest.skipUnless(config.extended, "part of extended tests") class BFDTunTestCase(VppTestCase): """ BFD over GRE tunnel """ diff --git a/test/test_bier.py b/test/test_bier.py index 2f649bbde53..7dd6a30f34b 100644 --- a/test/test_bier.py +++ b/test/test_bier.py @@ -2,7 +2,8 @@ import unittest -from framework import VppTestCase, VppTestRunner, running_extended_tests +from config import config +from framework import VppTestCase, VppTestRunner from vpp_ip import DpoProto from vpp_ip_route import VppIpRoute, VppRoutePath, \ VppMplsTable, VppIpMRoute, VppMRoutePath, VppIpTable, \ @@ -183,22 +184,22 @@ class TestBier(VppTestCase): for nhr in nh_routes: nhr.remove_vpp_config() - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_bier_midpoint_1024(self): """BIER midpoint BSL:1024""" self.bier_midpoint(BIERLength.BIER_LEN_1024, 128, 1024) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_bier_midpoint_512(self): """BIER midpoint BSL:512""" self.bier_midpoint(BIERLength.BIER_LEN_512, 64, 512) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_bier_midpoint_256(self): """BIER midpoint BSL:256""" self.bier_midpoint(BIERLength.BIER_LEN_256, 32, 256) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_bier_midpoint_128(self): """BIER midpoint BSL:128""" self.bier_midpoint(BIERLength.BIER_LEN_128, 16, 128) @@ -674,22 +675,22 @@ class TestBier(VppTestCase): self.assertEqual(rx[0][IP].src, "1.1.1.1") self.assertEqual(rx[0][IP].dst, "232.1.1.2") - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_bier_e2e_1024(self): """ BIER end-to-end BSL:1024""" self.bier_e2e(BIERLength.BIER_LEN_1024, 128, 1024) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_bier_e2e_512(self): """ BIER end-to-end BSL:512""" self.bier_e2e(BIERLength.BIER_LEN_512, 64, 512) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_bier_e2e_256(self): """ BIER end-to-end BSL:256""" self.bier_e2e(BIERLength.BIER_LEN_256, 32, 256) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_bier_e2e_128(self): """ BIER end-to-end BSL:128""" self.bier_e2e(BIERLength.BIER_LEN_128, 16, 128) diff --git a/test/test_bihash.py b/test/test_bihash.py index 2949d66750d..32eb4ff3a21 100644 --- a/test/test_bihash.py +++ b/test/test_bihash.py @@ -2,7 +2,8 @@ import unittest -from framework import VppTestCase, VppTestRunner, running_gcov_tests +from config import config +from framework import VppTestCase, VppTestRunner from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath @@ -53,7 +54,7 @@ class TestBihash(VppTestCase): self.logger.critical(error) self.assertNotIn('failed', error) - @unittest.skipUnless(running_gcov_tests, "part of code coverage tests") + @unittest.skipUnless(config.gcov, "part of code coverage tests") def test_bihash_coverage(self): """ Improve Code Coverage """ diff --git a/test/test_container.py b/test/test_container.py index 474805333c5..739aaaf915a 100644 --- a/test/test_container.py +++ b/test/test_container.py @@ -2,7 +2,8 @@ """ Container integration tests """ import unittest -from framework import VppTestCase, VppTestRunner, running_extended_tests +from config import config +from framework import VppTestCase, VppTestRunner from scapy.layers.l2 import Ether from scapy.packet import Raw from scapy.layers.inet import IP, UDP, TCP @@ -21,7 +22,7 @@ class Conn(L4_Conn): pass -@unittest.skipUnless(running_extended_tests, "part of extended tests") +@unittest.skipUnless(config.extended, "part of extended tests") class ContainerIntegrationTestCase(VppTestCase): """ Container integration extended testcases """ diff --git a/test/test_det44.py b/test/test_det44.py index d022cd53cd3..f137e760c38 100644 --- a/test/test_det44.py +++ b/test/test_det44.py @@ -4,7 +4,9 @@ import socket import struct import unittest import scapy.compat -from framework import VppTestCase, running_extended_tests +from time import sleep +from config import config +from framework import VppTestCase from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder from scapy.layers.inet import IP, TCP, UDP, ICMP from scapy.layers.inet import IPerror, UDPerror @@ -603,7 +605,7 @@ class TestDET44(VppTestCase): self.assertEqual(0, dms[0].ses_num) # TODO: ipfix needs to be separated from NAT base plugin - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_session_limit_per_user(self): """ Deterministic NAT maximum sessions per user limit """ self.vapi.det44_add_del_map(is_add=1, in_addr=self.pg0.remote_ip4, diff --git a/test/test_dhcp.py b/test/test_dhcp.py index e17b0049df7..a3b410c0595 100644 --- a/test/test_dhcp.py +++ b/test/test_dhcp.py @@ -5,7 +5,7 @@ import socket import struct import six -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from framework import tag_run_solo from vpp_neighbor import VppNeighbor from vpp_ip_route import find_route, VppIpTable diff --git a/test/test_dslite.py b/test/test_dslite.py index 2b4f4aacc9f..8f5995e61fa 100644 --- a/test/test_dslite.py +++ b/test/test_dslite.py @@ -6,7 +6,7 @@ import struct import random from framework import tag_fixme_vpp_workers -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner import scapy.compat from scapy.layers.inet import IP, TCP, UDP, ICMP diff --git a/test/test_flowprobe.py b/test/test_flowprobe.py index a7a09fca19b..5bafd39eceb 100644 --- a/test/test_flowprobe.py +++ b/test/test_flowprobe.py @@ -12,8 +12,9 @@ from scapy.layers.l2 import Ether from scapy.layers.inet import IP, TCP, UDP from scapy.layers.inet6 import IPv6 +from config import config from framework import tag_fixme_vpp_workers -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from framework import tag_run_solo from vpp_object import VppObject from vpp_pg_interface import CaptureTimeoutError @@ -827,7 +828,7 @@ class Datapath(MethodHolder): self.logger.info("FFP_TEST_FINISH_0002") -@unittest.skipUnless(running_extended_tests, "part of extended tests") +@unittest.skipUnless(config.extended, "part of extended tests") class DisableIPFIX(MethodHolder): """Disable IPFIX""" @@ -875,7 +876,7 @@ class DisableIPFIX(MethodHolder): self.logger.info("FFP_TEST_FINISH_0001") -@unittest.skipUnless(running_extended_tests, "part of extended tests") +@unittest.skipUnless(config.extended, "part of extended tests") class ReenableIPFIX(MethodHolder): """Re-enable IPFIX""" @@ -943,7 +944,7 @@ class ReenableIPFIX(MethodHolder): self.logger.info("FFP_TEST_FINISH_0001") -@unittest.skipUnless(running_extended_tests, "part of extended tests") +@unittest.skipUnless(config.extended, "part of extended tests") class DisableFP(MethodHolder): """Disable Flowprobe feature""" @@ -990,7 +991,7 @@ class DisableFP(MethodHolder): self.logger.info("FFP_TEST_FINISH_0001") -@unittest.skipUnless(running_extended_tests, "part of extended tests") +@unittest.skipUnless(config.extended, "part of extended tests") class ReenableFP(MethodHolder): """Re-enable Flowprobe feature""" diff --git a/test/test_igmp.py b/test/test_igmp.py index 8053bc3d544..f99bdb25c8c 100644 --- a/test/test_igmp.py +++ b/test/test_igmp.py @@ -8,7 +8,7 @@ from scapy.layers.inet import IP, IPOption from scapy.contrib.igmpv3 import IGMPv3, IGMPv3gr, IGMPv3mq, IGMPv3mr from framework import tag_fixme_vpp_workers -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from vpp_igmp import find_igmp_state, IGMP_FILTER, IgmpRecord, IGMP_MODE, \ IgmpSG, VppHostState, wait_for_igmp_event from vpp_ip_route import find_mroute, VppIpTable diff --git a/test/test_ikev2.py b/test/test_ikev2.py index 438a674977f..58a7ec3cd9a 100644 --- a/test/test_ikev2.py +++ b/test/test_ikev2.py @@ -13,6 +13,7 @@ from cryptography.hazmat.primitives.ciphers import ( ) from ipaddress import IPv4Address, IPv6Address, ip_address import unittest +from config import config from scapy.layers.ipsec import ESP from scapy.layers.inet import IP, UDP, Ether from scapy.layers.inet6 import IPv6 @@ -1380,7 +1381,7 @@ class Ikev2Params(object): if 'auth' in params and params['auth'] == 'rsa-sig': auth_method = 'rsa-sig' - work_dir = os.getenv('BR') + '/../src/plugins/ikev2/test/certs/' + work_dir = f"{config.vpp_ws_dir}/src/plugins/ikev2/test/certs/" self.vapi.ikev2_set_local_key( key_file=work_dir + params['server-key']) diff --git a/test/test_ipfix_export.py b/test/test_ipfix_export.py index 7718f423499..be7476e67cf 100644 --- a/test/test_ipfix_export.py +++ b/test/test_ipfix_export.py @@ -7,7 +7,7 @@ import unittest import time import re -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase from vpp_object import VppObject from vpp_pg_interface import CaptureTimeoutError from vpp_ip_route import VppIpRoute, VppRoutePath diff --git a/test/test_l2bd_learnlimit.py b/test/test_l2bd_learnlimit.py index 434a2922a64..91740590151 100644 --- a/test/test_l2bd_learnlimit.py +++ b/test/test_l2bd_learnlimit.py @@ -7,7 +7,7 @@ from scapy.packet import Raw from scapy.layers.l2 import Ether from scapy.layers.inet import IP, UDP -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from util import Host, ppp diff --git a/test/test_l2bd_learnlimit_bdenabled.py b/test/test_l2bd_learnlimit_bdenabled.py index 503a7d82dc4..fdbc3507eeb 100644 --- a/test/test_l2bd_learnlimit_bdenabled.py +++ b/test/test_l2bd_learnlimit_bdenabled.py @@ -7,7 +7,7 @@ from scapy.packet import Raw from scapy.layers.l2 import Ether from scapy.layers.inet import IP, UDP -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from util import Host, ppp diff --git a/test/test_l2bd_learnlimit_enabled.py b/test/test_l2bd_learnlimit_enabled.py index b323af62d69..906ae14b0bf 100644 --- a/test/test_l2bd_learnlimit_enabled.py +++ b/test/test_l2bd_learnlimit_enabled.py @@ -7,7 +7,7 @@ from scapy.packet import Raw from scapy.layers.l2 import Ether from scapy.layers.inet import IP, UDP -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from util import Host, ppp diff --git a/test/test_l2bd_multi_instance.py b/test/test_l2bd_multi_instance.py index b51bb5e6c81..c4692d686b0 100644 --- a/test/test_l2bd_multi_instance.py +++ b/test/test_l2bd_multi_instance.py @@ -69,7 +69,7 @@ from scapy.packet import Raw from scapy.layers.l2 import Ether from scapy.layers.inet import IP, UDP -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from util import Host, ppp diff --git a/test/test_mactime.py b/test/test_mactime.py index 85ded33d158..1eaeeb53a9d 100644 --- a/test/test_mactime.py +++ b/test/test_mactime.py @@ -2,7 +2,8 @@ import unittest -from framework import VppTestCase, VppTestRunner, running_gcov_tests +from config import config +from framework import VppTestCase, VppTestRunner from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath @@ -31,7 +32,7 @@ class TestMactime(VppTestCase): self.logger.critical(error) self.assertNotIn('FAILED', error) - @unittest.skipUnless(running_gcov_tests, "part of code coverage tests") + @unittest.skipUnless(config.gcov, "part of code coverage tests") def test_mactime_unittest(self): """ Mactime Plugin Code Coverage Test """ cmds = ["loopback create", diff --git a/test/test_memif.py b/test/test_memif.py index bf1c10a0a8a..26e44f29485 100644 --- a/test/test_memif.py +++ b/test/test_memif.py @@ -4,7 +4,7 @@ import unittest from scapy.layers.l2 import Ether from scapy.layers.inet import IP, ICMP -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from framework import tag_run_solo from remote_test import RemoteClass, RemoteVppTestCase from vpp_memif import remove_all_memif_vpp_config, \ diff --git a/test/test_mpcap.py b/test/test_mpcap.py index d77b543799f..bfa4385ed1a 100644 --- a/test/test_mpcap.py +++ b/test/test_mpcap.py @@ -2,7 +2,7 @@ import unittest -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath import os diff --git a/test/test_nat64.py b/test/test_nat64.py index ccd81ef9145..c51adac09de 100644 --- a/test/test_nat64.py +++ b/test/test_nat64.py @@ -8,8 +8,9 @@ import unittest from io import BytesIO import scapy.compat +from config import config from framework import tag_fixme_vpp_workers -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder from scapy.data import IP_PROTOS from scapy.layers.inet import IP, TCP, UDP, ICMP @@ -1647,7 +1648,7 @@ class TestNAT64(VppTestCase): addresses = self.vapi.nat64_pool_addr_dump() self.assertEqual(0, len(addresses)) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_ipfix_max_bibs_sessions(self): """ IPFIX logging maximum session and BIB entries exceeded """ max_bibs = 1280 diff --git a/test/test_nat66.py b/test/test_nat66.py index e3236216bea..02b2882a28b 100644 --- a/test/test_nat66.py +++ b/test/test_nat66.py @@ -8,7 +8,7 @@ import unittest from io import BytesIO import scapy.compat -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder from scapy.all import bind_layers, Packet, ByteEnumField, ShortField, \ IPField, IntField, LongField, XByteField, FlagsField, FieldLenField, \ diff --git a/test/test_offload.py b/test/test_offload.py index 9f6aec65482..ae5a5b3c6dc 100644 --- a/test/test_offload.py +++ b/test/test_offload.py @@ -2,7 +2,7 @@ import unittest -from framework import VppTestCase, VppTestRunner, running_gcov_tests +from framework import VppTestCase, VppTestRunner from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath diff --git a/test/test_pcap.py b/test/test_pcap.py index c058c5467af..5bfac523e43 100644 --- a/test/test_pcap.py +++ b/test/test_pcap.py @@ -3,7 +3,7 @@ import os import unittest -from framework import VppTestCase, VppTestRunner, running_gcov_tests +from framework import VppTestCase, VppTestRunner from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath diff --git a/test/test_quic.py b/test/test_quic.py index ea367c0ba14..339557d4419 100644 --- a/test/test_quic.py +++ b/test/test_quic.py @@ -5,9 +5,9 @@ import unittest import os import subprocess import signal +from config import config from framework import tag_fixme_vpp_workers -from framework import VppTestCase, VppTestRunner, running_extended_tests, \ - Worker +from framework import VppTestCase, VppTestRunner, Worker from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath @@ -15,11 +15,11 @@ class QUICAppWorker(Worker): """ QUIC Test Application Worker """ process = None - def __init__(self, build_dir, appname, executable_args, logger, role, + def __init__(self, appname, executable_args, logger, role, testcase, env=None, *args, **kwargs): if env is None: env = {} - app = "%s/bin/%s" % (build_dir, appname) + app = f"{config.vpp_build_dir}/vpp/bin/{appname}" self.args = [app] + executable_args self.role = role self.wait_for_gdb = 'wait-for-gdb' @@ -57,11 +57,7 @@ class QUICTestCase(VppTestCase): def setUp(self): super(QUICTestCase, self).setUp() - var = "VPP_BUILD_DIR" - self.build_dir = os.getenv(var, None) - if self.build_dir is None: - raise Exception("Environment variable `%s' not set" % var) - self.vppDebug = 'vpp_debug' in self.build_dir + self.vppDebug = 'vpp_debug' in config.vpp_build_dir self.create_loopback_interfaces(2) self.uri = "quic://%s/1234" % self.loop0.local_ip4 @@ -218,7 +214,6 @@ class QUICEchoExtTestCase(QUICTestCase): def server(self, *args): _args = self.server_echo_test_args + list(args) self.worker_server = QUICAppWorker( - self.build_dir, self.app, _args, self.logger, @@ -230,7 +225,6 @@ class QUICEchoExtTestCase(QUICTestCase): def client(self, *args): _args = self.client_echo_test_args + list(args) self.worker_client = QUICAppWorker( - self.build_dir, self.app, _args, self.logger, @@ -291,7 +285,7 @@ class QUICEchoExtTransferBigTestCase(QUICEchoExtTestCase): test_bytes = '' timeout = 60 - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_quic_ext_transfer_big(self): """QUIC external transfer, big stream""" self.server("TX=0", "RX=2G") @@ -302,7 +296,7 @@ class QUICEchoExtTransferBigTestCase(QUICEchoExtTestCase): class QUICEchoExtQcloseRxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Qclose Rx Test Case""" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_qclose_rx(self): """QUIC external transfer, rx close""" @@ -314,7 +308,7 @@ class QUICEchoExtQcloseRxTestCase(QUICEchoExtTestCase): class QUICEchoExtQcloseTxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Qclose Tx Test Case""" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_qclose_tx(self): """QUIC external transfer, tx close""" @@ -327,7 +321,7 @@ class QUICEchoExtQcloseTxTestCase(QUICEchoExtTestCase): class QUICEchoExtEarlyQcloseRxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Early Qclose Rx Test Case""" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_early_qclose_rx(self): """QUIC external transfer, early rx close""" @@ -340,7 +334,7 @@ class QUICEchoExtEarlyQcloseRxTestCase(QUICEchoExtTestCase): class QUICEchoExtEarlyQcloseTxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Early Qclose Tx Test Case""" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_early_qclose_tx(self): """QUIC external transfer, early tx close""" @@ -353,7 +347,7 @@ class QUICEchoExtEarlyQcloseTxTestCase(QUICEchoExtTestCase): class QUICEchoExtScloseRxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Sclose Rx Test Case""" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_sclose_rx(self): """QUIC external transfer, rx stream close""" @@ -365,7 +359,7 @@ class QUICEchoExtScloseRxTestCase(QUICEchoExtTestCase): class QUICEchoExtScloseTxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Sclose Tx Test Case""" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_sclose_tx(self): """QUIC external transfer, tx stream close""" @@ -377,7 +371,7 @@ class QUICEchoExtScloseTxTestCase(QUICEchoExtTestCase): class QUICEchoExtEarlyScloseRxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Early Sclose Rx Test Case""" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_early_sclose_rx(self): """QUIC external transfer, early rx stream close""" @@ -390,7 +384,7 @@ class QUICEchoExtEarlyScloseRxTestCase(QUICEchoExtTestCase): class QUICEchoExtEarlyScloseTxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Early Sclose Tx Test Case""" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_early_sclose_tx(self): """QUIC external transfer, early tx stream close""" @@ -420,7 +414,7 @@ class QUICEchoExtServerStreamBigTestCase(QUICEchoExtTestCase): test_bytes = '' timeout = 60 - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_quic_ext_transfer_server_stream_big(self): """QUIC external server transfer, big stream""" self.server("TX=2G", "RX=0") @@ -432,7 +426,7 @@ class QUICEchoExtServerStreamQcloseRxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Server Stream Qclose Rx Test Case""" quic_setup = "serverstream" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_server_stream_qclose_rx(self): """QUIC external server transfer, rx close""" @@ -445,7 +439,7 @@ class QUICEchoExtServerStreamQcloseTxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Server Stream Qclose Tx Test Case""" quic_setup = "serverstream" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_server_stream_qclose_tx(self): """QUIC external server transfer, tx close""" @@ -459,7 +453,7 @@ class QUICEchoExtServerStreamEarlyQcloseRxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Server Stream Early Qclose Rx Test Case""" quic_setup = "serverstream" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_server_stream_early_qclose_rx(self): """QUIC external server transfer, early rx close""" @@ -473,7 +467,7 @@ class QUICEchoExtServerStreamEarlyQcloseTxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Server Stream Early Qclose Tx Test Case""" quic_setup = "serverstream" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_server_stream_early_qclose_tx(self): """QUIC external server transfer, early tx close""" @@ -487,7 +481,7 @@ class QUICEchoExtServerStreamScloseRxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Server Stream Sclose Rx Test Case""" quic_setup = "serverstream" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_server_stream_sclose_rx(self): """QUIC external server transfer, rx stream close""" @@ -500,7 +494,7 @@ class QUICEchoExtServerStreamScloseTxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Server Stream Sclose Tx Test Case""" quic_setup = "serverstream" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_server_stream_sclose_tx(self): """QUIC external server transfer, tx stream close""" @@ -513,7 +507,7 @@ class QUICEchoExtServerStreamEarlyScloseRxTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Server Stream Early Sclose Rx Test Case""" quic_setup = "serverstream" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_server_stream_early_sclose_rx(self): """QUIC external server transfer, early rx stream close""" @@ -527,7 +521,7 @@ class QUICEchoExtServerStreamEarlyScloseTxTestCase(QUICEchoExtTestCase): """QUIC Echo Ext Transfer Server Stream Early Sclose Tx Test Case""" quic_setup = "serverstream" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_server_stream_early_sclose_tx(self): """QUIC external server transfer, early tx stream close""" @@ -541,7 +535,7 @@ class QUICEchoExtServerStreamWorkersTestCase(QUICEchoExtTestCase): """QUIC Echo External Transfer Server Stream MultiWorker Test Case""" quic_setup = "serverstream" - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("testcase under development") def test_quic_ext_transfer_server_stream_multi_workers(self): """QUIC external server transfer, multi-worker""" diff --git a/test/test_trace_filter.py b/test/test_trace_filter.py index a37d4b996c8..fa0d74812f5 100644 --- a/test/test_trace_filter.py +++ b/test/test_trace_filter.py @@ -2,7 +2,7 @@ import unittest -from framework import VppTestCase, VppTestRunner, running_extended_tests +from framework import VppTestCase, VppTestRunner from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath from scapy.contrib.geneve import GENEVE diff --git a/test/test_vapi.py b/test/test_vapi.py index 106c84a9bad..5a04abe4de7 100644 --- a/test/test_vapi.py +++ b/test/test_vapi.py @@ -4,6 +4,7 @@ import unittest import os import signal +from config import config from framework import VppTestCase, VppTestRunner, Worker @@ -20,11 +21,7 @@ class VAPITestCase(VppTestCase): def test_vapi_c(self): """ run C VAPI tests """ - var = "VPP_BUILD_DIR" - build_dir = os.getenv(var, None) - self.assertIsNotNone(build_dir, - "Environment variable `%s' not set" % var) - executable = f"{build_dir}/bin/vapi_c_test" + executable = f"{config.vpp_build_dir}/vpp/bin/vapi_c_test" worker = Worker([executable, "vapi client", self.get_api_segment_prefix()], self.logger) worker.start() @@ -49,11 +46,7 @@ class VAPITestCase(VppTestCase): def test_vapi_cpp(self): """ run C++ VAPI tests """ - var = "VPP_BUILD_DIR" - build_dir = os.getenv(var, None) - self.assertIsNotNone(build_dir, - "Environment variable `%s' not set" % var) - executable = f"{build_dir}/bin/vapi_cpp_test" + executable = f"{config.vpp_build_dir}/vpp/bin/vapi_cpp_test" worker = Worker([executable, "vapi client", self.get_api_segment_prefix()], self.logger) worker.start() diff --git a/test/test_vcl.py b/test/test_vcl.py index e40e416ebfa..3875114c3b2 100644 --- a/test/test_vcl.py +++ b/test/test_vcl.py @@ -6,8 +6,8 @@ import os import subprocess import signal import glob -from framework import VppTestCase, VppTestRunner, running_extended_tests, \ - Worker +from config import config +from framework import VppTestCase, VppTestRunner, Worker from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath, FibPathProto iperf3 = '/usr/bin/iperf3' @@ -32,12 +32,10 @@ class VCLAppWorker(Worker): class LibraryNotFound(Exception): pass - def __init__(self, build_dir, appname, executable_args, logger, env=None, + def __init__(self, appname, executable_args, logger, env=None, role=None, *args, **kwargs): self.role = role - vpp_install_path = os.getenv('VPP_INSTALL_PATH') - - vcl_ldpreload_glob = "{}/**/{}".format(vpp_install_path, self.libname) + vcl_ldpreload_glob = f"{config.vpp_install_dir}/**/{self.libname}" vcl_ldpreload_so = glob.glob(vcl_ldpreload_glob, recursive=True) if len(vcl_ldpreload_so) < 1: @@ -52,10 +50,10 @@ class VCLAppWorker(Worker): app = appname env.update({'LD_PRELOAD': vcl_ldpreload_so}) elif "sock" in appname: - app = "%s/bin/%s" % (build_dir, appname) + app = f"{config.vpp_build_dir}/vpp/bin/{appname}" env.update({'LD_PRELOAD': vcl_ldpreload_so}) else: - app = "%s/bin/%s" % (build_dir, appname) + app = f"{config.vpp_build_dir}/vpp/bin/{appname}" self.args = [app] + executable_args super(VCLAppWorker, self).__init__(self.args, logger, env, *args, **kwargs) @@ -77,11 +75,7 @@ class VCLTestCase(VppTestCase): super(VCLTestCase, cls).tearDownClass() def setUp(self): - var = "VPP_BUILD_DIR" - self.build_dir = os.getenv(var, None) - if self.build_dir is None: - raise EnvironmentError("Environment variable `%s' not set" % var) - self.vppDebug = 'vpp_debug' in self.build_dir + self.vppDebug = 'vpp_debug' in config.vpp_install_dir self.server_addr = "127.0.0.1" self.server_port = "22000" self.server_args = [self.server_port] @@ -132,13 +126,13 @@ class VCLTestCase(VppTestCase): self.vcl_app_env = {'VCL_APP_SCOPE_LOCAL': "true"} self.update_vcl_app_env("", "", self.sapi_server_sock) - worker_server = VCLAppWorker(self.build_dir, server_app, server_args, + worker_server = VCLAppWorker(server_app, server_args, self.logger, self.vcl_app_env, "server") worker_server.start() self.sleep(self.pre_test_sleep) self.update_vcl_app_env("", "", self.sapi_client_sock) - worker_client = VCLAppWorker(self.build_dir, client_app, client_args, + worker_client = VCLAppWorker(client_app, client_args, self.logger, self.vcl_app_env, "client") worker_client.start() worker_client.join(self.timeout) @@ -240,13 +234,13 @@ class VCLTestCase(VppTestCase): self.vcl_app_env = {'VCL_APP_SCOPE_GLOBAL': "true"} self.update_vcl_app_env("1", "1234", self.sapi_server_sock) - worker_server = VCLAppWorker(self.build_dir, server_app, server_args, + worker_server = VCLAppWorker(server_app, server_args, self.logger, self.vcl_app_env, "server") worker_server.start() self.sleep(self.pre_test_sleep) self.update_vcl_app_env("2", "5678", self.sapi_client_sock) - worker_client = VCLAppWorker(self.build_dir, client_app, client_args, + worker_client = VCLAppWorker(client_app, client_args, self.logger, self.vcl_app_env, "client") worker_client.start() worker_client.join(self.timeout) @@ -328,14 +322,13 @@ class LDPCutThruTestCase(VCLTestCase): self.logger.debug(self.vapi.cli("show session verbose 2")) self.logger.debug(self.vapi.cli("show app mq")) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_ldp_cut_thru_echo(self): """ run LDP cut thru echo test """ self.cut_thru_test("sock_test_server", self.server_args, "sock_test_client", self.client_echo_test_args) - @unittest.skipUnless(_have_iperf3, "'%s' not found, Skipping.") def test_ldp_cut_thru_iperf3(self): """ run LDP cut thru iperf3 test """ @@ -343,7 +336,7 @@ class LDPCutThruTestCase(VCLTestCase): self.cut_thru_test(iperf3, self.server_iperf3_args, iperf3, self.client_iperf3_args) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_ldp_cut_thru_uni_dir_nsock(self): """ run LDP cut thru uni-directional (multiple sockets) test """ @@ -352,7 +345,7 @@ class LDPCutThruTestCase(VCLTestCase): "sock_test_client", self.client_uni_dir_nsock_test_args) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("sock test apps need to be improved") def test_ldp_cut_thru_bi_dir_nsock(self): """ run LDP cut thru bi-directional (multiple sockets) test """ @@ -569,7 +562,7 @@ class VCLThruHostStackQUIC(VCLTestCase): self.loop0.local_ip4, self.server_port] - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_vcl_thru_host_stack_quic_uni_dir(self): """ run VCL thru host stack uni-directional QUIC test """ @@ -867,6 +860,7 @@ class LDPIpv6CutThruTestCase(VCLTestCase): super(LDPIpv6CutThruTestCase, self).tearDown() self.cut_thru_tear_down() + @unittest.skipUnless(config.extended, "part of extended tests") def test_ldp_ipv6_cut_thru_echo(self): """ run LDP IPv6 cut thru echo test """ @@ -876,7 +870,6 @@ class LDPIpv6CutThruTestCase(VCLTestCase): self.client_ipv6_echo_test_args) @unittest.skipUnless(_have_iperf3, "'%s' not found, Skipping.") - @unittest.skipUnless(running_extended_tests, "part of extended tests") def test_ldp_ipv6_cut_thru_iperf3(self): """ run LDP IPv6 cut thru iperf3 test """ @@ -884,7 +877,7 @@ class LDPIpv6CutThruTestCase(VCLTestCase): self.cut_thru_test(iperf3, self.server_ipv6_iperf3_args, iperf3, self.client_ipv6_iperf3_args) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_ldp_ipv6_cut_thru_uni_dir_nsock(self): """ run LDP IPv6 cut thru uni-directional (multiple sockets) test """ @@ -893,7 +886,7 @@ class LDPIpv6CutThruTestCase(VCLTestCase): "sock_test_client", self.client_ipv6_uni_dir_nsock_test_args) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") @unittest.skip("sock test apps need to be improved") def test_ldp_ipv6_cut_thru_bi_dir_nsock(self): """ run LDP IPv6 cut thru bi-directional (multiple sockets) test """ @@ -955,7 +948,7 @@ class VCLIpv6CutThruTestCase(VCLTestCase): "vcl_test_client", self.client_ipv6_echo_test_args) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_vcl_ipv6_cut_thru_uni_dir_nsock(self): """ run VCL IPv6 cut thru uni-directional (multiple sockets) test """ @@ -964,7 +957,7 @@ class VCLIpv6CutThruTestCase(VCLTestCase): "vcl_test_client", self.client_ipv6_uni_dir_nsock_test_args) - @unittest.skipUnless(running_extended_tests, "part of extended tests") + @unittest.skipUnless(config.extended, "part of extended tests") def test_vcl_ipv6_cut_thru_bi_dir_nsock(self): """ run VCL IPv6 cut thru bi-directional (multiple sockets) test """ diff --git a/test/test_vlib.py b/test/test_vlib.py index 31fb72990f0..1d5a3eb2594 100644 --- a/test/test_vlib.py +++ b/test/test_vlib.py @@ -4,11 +4,12 @@ import unittest import pexpect import time import signal -from framework import VppTestCase, VppTestRunner, running_extended_tests -from framework import running_gcov_tests +from config import config +from framework import VppTestCase, VppTestRunner from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath +@unittest.skipUnless(config.gcov, "part of code coverage tests") class TestVlib(VppTestCase): """ Vlib Unit Test Cases """ vpp_worker_count = 1 @@ -27,7 +28,6 @@ class TestVlib(VppTestCase): def tearDown(self): super(TestVlib, self).tearDown() - @unittest.skipUnless(running_gcov_tests, "part of code coverage tests") def test_vlib_main_unittest(self): """ Vlib main.c Code Coverage Test """ @@ -68,7 +68,6 @@ class TestVlib(VppTestCase): else: self.logger.info(cmd + " FAIL retval " + str(r.retval)) - @unittest.skipUnless(running_gcov_tests, "part of code coverage tests") def test_vlib_node_cli_unittest(self): """ Vlib node_cli.c Code Coverage Test """ @@ -113,7 +112,6 @@ class TestVlib(VppTestCase): else: self.logger.info(cmd + " FAIL retval " + str(r.retval)) - @unittest.skipUnless(running_gcov_tests, "part of code coverage tests") def test_vlib_buffer_c_unittest(self): """ Vlib buffer.c Code Coverage Test """ @@ -169,7 +167,6 @@ class TestVlib(VppTestCase): else: self.logger.info(cmd + " FAIL retval " + str(r.retval)) - @unittest.skipUnless(running_gcov_tests, "part of code coverage tests") def test_vlib_format_unittest(self): """ Vlib format.c Code Coverage Test """ @@ -187,7 +184,6 @@ class TestVlib(VppTestCase): else: self.logger.info(cmd + " FAIL retval " + str(r.retval)) - @unittest.skipUnless(running_gcov_tests, "part of code coverage tests") def test_vlib_main_unittest(self): """ Private Binary API Segment Test (takes 70 seconds) """ diff --git a/test/test_vppinfra.py b/test/test_vppinfra.py index 8b6ec965fea..36cd55b3550 100644 --- a/test/test_vppinfra.py +++ b/test/test_vppinfra.py @@ -2,8 +2,7 @@ import unittest -from framework import VppTestCase, VppTestRunner, running_extended_tests -from framework import running_gcov_tests +from framework import VppTestCase, VppTestRunner class TestVppinfra(VppTestCase): diff --git a/test/test_vrrp.py b/test/test_vrrp.py index 241ca0d3cf4..1f28cf3d18d 100644 --- a/test/test_vrrp.py +++ b/test/test_vrrp.py @@ -23,7 +23,8 @@ from scapy.layers.inet6 import IPv6, ipv6nh, IPv6ExtHdrHopByHop, \ from scapy.contrib.igmpv3 import IGMPv3, IGMPv3mr, IGMPv3gr from scapy.layers.vrrp import IPPROTO_VRRP, VRRPv3 from scapy.utils6 import in6_getnsma, in6_getnsmac -from framework import VppTestCase, VppTestRunner, running_extended_tests +from config import config +from framework import VppTestCase, VppTestRunner from util import ip6_normalize VRRP_VR_FLAG_PREEMPT = 1 @@ -215,7 +216,7 @@ class VppVRRPVirtualRouter(VppObject): return pkt -@unittest.skipUnless(running_extended_tests, "part of extended tests") +@unittest.skipUnless(config.extended, "part of extended tests") class TestVRRP4(VppTestCase): """ IPv4 VRRP Test Case """ @@ -748,7 +749,7 @@ class TestVRRP4(VppTestCase): self.assertEqual(rx[VRRPv3].addrlist, [vip]) -@unittest.skipUnless(running_extended_tests, "part of extended tests") +@unittest.skipUnless(config.extended, "part of extended tests") class TestVRRP6(VppTestCase): """ IPv6 VRRP Test Case """ diff --git a/test/test_vxlan_gpe.py b/test/test_vxlan_gpe.py index 28c31ccd3bb..828b5fc30cd 100644 --- a/test/test_vxlan_gpe.py +++ b/test/test_vxlan_gpe.py @@ -3,7 +3,8 @@ import socket from util import ip4_range import unittest -from framework import VppTestCase, VppTestRunner, running_extended_tests +from config import config +from framework import VppTestCase, VppTestRunner from template_bd import BridgeDomain from scapy.layers.l2 import Ether @@ -17,7 +18,7 @@ from vpp_vxlan_gpe_tunnel import VppVxlanGpeTunnel from vpp_ip import INVALID_INDEX -@unittest.skipUnless(running_extended_tests, "part of extended tests") +@unittest.skipUnless(config.extended, "part of extended tests") class TestVxlanGpe(BridgeDomain, VppTestCase): """ VXLAN-GPE Test Case """ diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py index 1c5b6c5084f..c0e88f826e0 100644 --- a/test/vpp_papi_provider.py +++ b/test/vpp_papi_provider.py @@ -10,6 +10,7 @@ import time from collections import deque import queue from six import moves, iteritems +from config import config from vpp_papi import VPPApiClient, mac_pton from hook import Hook from vpp_ip_route import MPLS_IETF_MAX_LABEL, MPLS_LABEL_INVALID @@ -135,7 +136,7 @@ class VppPapiProvider(object): # install_dir is a class attribute. We need to set it before # calling the constructor. - VPPApiClient.apidir = os.getenv('VPP_INSTALL_PATH') + VPPApiClient.apidir = config.vpp_install_dir self.vpp = VPPApiClient(logger=test_class.logger, read_timeout=read_timeout, diff --git a/test/vpp_pg_interface.py b/test/vpp_pg_interface.py index 567b81de7c0..01f10fccfc9 100644 --- a/test/vpp_pg_interface.py +++ b/test/vpp_pg_interface.py @@ -5,6 +5,7 @@ import struct import time from traceback import format_exc, format_stack +from config import config import scapy.compat from scapy.utils import wrpcap, rdpcap, PcapReader from scapy.plist import PacketList @@ -133,9 +134,20 @@ class VppPGInterface(VppInterface): self._cap_name = "pcap%u-sw_if_index-%s" % ( self.pg_index, self.sw_if_index) - def rename_old_pcap_file(self, path, counter): + def handle_old_pcap_file(self, path, counter): filename = os.path.basename(path) + + if not config.keep_pcaps: + try: + self.test.logger.debug(f"Removing {path}") + os.remove(path) + except OSError: + self.test.logger.debug(f"OSError: Could not remove {path}") + return + + # keep try: + if os.path.isfile(path): name = "%s/history.[timestamp:%f].[%s-counter:%04d].%s" % \ (self.test.tempdir, @@ -143,8 +155,7 @@ class VppPGInterface(VppInterface): self.name, counter, filename) - self.test.logger.debug("Renaming %s->%s" % - (path, name)) + self.test.logger.debug("Renaming %s->%s" % (path, name)) os.rename(path, name) except OSError: self.test.logger.debug("OSError: Could not rename %s %s" % @@ -157,7 +168,7 @@ class VppPGInterface(VppInterface): """ # disable the capture to flush the capture self.disable_capture() - self.rename_old_pcap_file(self.out_path, self.out_history_counter) + self.handle_old_pcap_file(self.out_path, self.out_history_counter) # FIXME this should be an API, but no such exists atm self.test.vapi.cli(self.capture_cli) self._pcap_reader = None |