diff options
| -rw-r--r-- | .gitignore | 8 | ||||
| -rw-r--r-- | Makefile | 45 | ||||
| -rw-r--r-- | docs/developer/build-run-debug/code_coverage.rst | 45 | ||||
| -rw-r--r-- | docs/developer/build-run-debug/index.rst | 1 | ||||
| -rw-r--r-- | docs/developer/extras/index.rst | 1 | ||||
| l--------- | docs/developer/extras/lcov.rst | 1 | ||||
| -rw-r--r-- | docs/spelling_wordlist.txt | 1 | ||||
| -rw-r--r-- | extras/lcov/README.rst | 48 | ||||
| -rwxr-xr-x | extras/lcov/lcov_post | 12 | ||||
| -rwxr-xr-x | extras/lcov/lcov_prep | 7 | ||||
| -rw-r--r-- | src/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/vlib/unix/main.c | 11 | ||||
| -rw-r--r-- | test/Makefile | 48 |
13 files changed, 116 insertions, 113 deletions
diff --git a/.gitignore b/.gitignore index ec9ee7b29aa..bcfee43fd49 100644 --- a/.gitignore +++ b/.gitignore @@ -16,14 +16,8 @@ /build-root/*.tar.xz /build-root/*.changes /build-root/rpmbuild/ -/build-root/test-doc/ -/build-root/test-cov/ +/build-root/test-coverage/ /build-root/test/ -/test/run/ -/test/build/ -/test/coverage/ -/test/vapi_test/ -/test/doc/build/ /build-config.mk /build/external/*.tar.gz /build/external/*.tar.xz @@ -65,7 +65,7 @@ endif DEB_DEPENDS = curl build-essential autoconf automake ccache DEB_DEPENDS += debhelper dkms git libtool libapr1-dev dh-python DEB_DEPENDS += libconfuse-dev git-review exuberant-ctags cscope pkg-config -DEB_DEPENDS += lcov chrpath autoconf libnuma-dev +DEB_DEPENDS += gcovr lcov chrpath autoconf libnuma-dev DEB_DEPENDS += python3-all python3-setuptools check DEB_DEPENDS += libffi-dev python3-ply DEB_DEPENDS += cmake ninja-build uuid-dev python3-jsonschema python3-yaml @@ -426,7 +426,7 @@ rebuild-release: wipe-release build-release export TEST_DIR ?= $(WS_ROOT)/test define test - $(if $(filter-out $(2),retest),make -C $(BR) PLATFORM=vpp TAG=$(1) vpp-install,) + $(if $(filter-out $(2),retest),make -C $(BR) PLATFORM=vpp TAG=$(1) CC=$(CC) vpp-install,) $(eval libs:=lib lib64) make -C test \ VPP_BUILD_DIR=$(BR)/build-$(1)-native/vpp \ @@ -449,10 +449,28 @@ test: test-debug: $(call test,vpp_debug,test) -.PHONY: test-gcov -test-gcov: +.PHONY: test-cov +test-cov: + $(eval CC=gcc) + $(eval TEST_GCOV=1) + $(call test,vpp_gcov,cov) + +.PHONY: test-cov-build +test-cov-build: + $(eval CC=gcc) + $(eval TEST_GCOV=1) $(call test,vpp_gcov,test) +.PHONY: test-cov-prep +test-cov-prep: + $(eval CC=gcc) + $(call test,vpp_gcov,cov-prep) + +.PHONY: test-cov-post +test-cov-post: + $(eval CC=gcc) + $(call test,vpp_gcov,cov-post) + .PHONY: test-all test-all: $(eval EXTENDED_TESTS=1) @@ -463,6 +481,13 @@ test-all-debug: $(eval EXTENDED_TESTS=1) $(call test,vpp_debug,test) +.PHONY: test-all-cov +test-all-cov: + $(eval CC=gcc) + $(eval TEST_GCOV=1) + $(eval EXTENDED_TESTS=1) + $(call test,vpp_gcov,test) + .PHONY: papi-wipe papi-wipe: test-wipe-papi $(call banner,"This command is deprecated. Please use 'test-wipe-papi'") @@ -487,8 +512,10 @@ test-shell: test-shell-debug: $(call test,vpp_debug,shell) -.PHONY: test-shell-gcov -test-shell-gcov: +.PHONY: test-shell-cov +test-shell-cov: + $(eval CC=gcc) + $(eval TEST_GCOV=1) $(call test,vpp_gcov,shell) .PHONY: test-dep @@ -505,13 +532,9 @@ test-wipe-doc: @echo "make test-wipe-doc is DEPRECATED" sleep 300 -.PHONY: test-cov -test-cov: - $(eval EXTENDED_TESTS=1) - $(call test,vpp_gcov,cov) - .PHONY: test-wipe-cov test-wipe-cov: + $(call make,$(PLATFORM)_gcov,$(addsuffix -wipe,$(TARGETS))) @make -C test wipe-cov .PHONY: test-wipe-all diff --git a/docs/developer/build-run-debug/code_coverage.rst b/docs/developer/build-run-debug/code_coverage.rst new file mode 100644 index 00000000000..efe750cda21 --- /dev/null +++ b/docs/developer/build-run-debug/code_coverage.rst @@ -0,0 +1,45 @@ +.. _lcov_code_coverage: + +Code coverage with lcov +======================= + +Prerequisites +------------- + +Ensure required packages are installed: + +:: + + $ make install-deps + +The Linux gcov and lcov tools can be fussy about gcc / g++ compiler +versions. As of this writing, Ubuntu 22.04 gcov / lcov works with +the latest gcc version (``11.3.0``). + +Generate coverage for a test case +--------------------------------- + +As a first run, in order to generate the coverage report of +a specific plugin or test, run for example + +:: + + $ make test-cov TEST=fib + +Then open the file ``.build-root/test-coverage/html/index.html`` in a Chrome browser. + +Improving test coverage +----------------------- + +When doing modifications on the test cases, you can run + +:: + + # This will run the test & report the result in the coverage data + $ make test-cov-build TEST=fib + + # This will generate the html report with the current coverage data + $ make test-cov-post + + # To reset the coverage data use + $ make test-cov-prep diff --git a/docs/developer/build-run-debug/index.rst b/docs/developer/build-run-debug/index.rst index f8bfeab0bf8..c99615ffcf5 100644 --- a/docs/developer/build-run-debug/index.rst +++ b/docs/developer/build-run-debug/index.rst @@ -12,3 +12,4 @@ Build, Run & Debug testing_vpp gdb_examples cross_compile_macos + code_coverage diff --git a/docs/developer/extras/index.rst b/docs/developer/extras/index.rst index a0f2d9b876a..2ee96be768e 100644 --- a/docs/developer/extras/index.rst +++ b/docs/developer/extras/index.rst @@ -7,7 +7,6 @@ VPP extra tools .. toctree:: :maxdepth: 2 - lcov snap strongswan vpp_config diff --git a/docs/developer/extras/lcov.rst b/docs/developer/extras/lcov.rst deleted file mode 120000 index 9f8de245ea6..00000000000 --- a/docs/developer/extras/lcov.rst +++ /dev/null @@ -1 +0,0 @@ -../../../extras/lcov/README.rst
\ No newline at end of file diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 80db33a354e..d28c6df56ca 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -397,7 +397,6 @@ Gerrit gethostbyname gettingsources gettingstarted -ggcov gid Gigabit GigE diff --git a/extras/lcov/README.rst b/extras/lcov/README.rst deleted file mode 100644 index 3dab9510215..00000000000 --- a/extras/lcov/README.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. _lcov_code_coverage: - -Code coverage with lcov -======================= - -Prerequisites -------------- - -The Linux gcov and lcov tools are fussy about gcc / g++ compiler -versions. As of this writing, Ubuntu 18.04 gcov / lcov work with these -toolchain versions: - -$ gcc –version gcc (Ubuntu 7.4.0-1ubuntu1~18.04.1) 7.4.0 $ g++ –version -g++ (Ubuntu 8.3.0-6ubuntu1~18.04.1) 8.3.0 - -Refer to -https://askubuntu.com/questions/26498/how-to-choose-the-default-gcc-and-g-version -for information on how to install multiple gcc / g++ versions, and -switch between them. - -You’ll need to install the following additional packages: - -$ sudo apt-get install gcovr ggcov lcov - -Compile an instrumented vpp image ---------------------------------- - -Two ways: - -$ cd $ make test-gcov $ ## interrupt compilation after building the -image - -or $ cd /build-root $ make PLATFORM=vpp TAG=vpp_gcov vpp-install - -Initialize the lcov database ----------------------------- - -$ cd $ ./extras/lcov/lcov_prep $ make test-gcov or make TEST=my_test -test-gcov $ # repeat or vary as desired to increase reported coverage $ -# Generate the report: $ ./extras/lcov/lcov_post - -You can run vpp manually, do anything you like. Results are cumulative -until you re-run the “prep” script. - -Look at the results -------------------- - -Point a browser at file:////build-root/html/index.html diff --git a/extras/lcov/lcov_post b/extras/lcov/lcov_post deleted file mode 100755 index 1aaf9865d24..00000000000 --- a/extras/lcov/lcov_post +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -e - -cd build-root -rm -rf html -mkdir html -lcov --no-checksum --directory . --capture --output-file out.info -lcov --remove out.info \ - "/usr/include/*" "*/build-root/*" "/opt/*" "/usr/lib/*" \ - -o filtered.info -genhtml filtered.info -o html diff --git a/extras/lcov/lcov_prep b/extras/lcov/lcov_prep deleted file mode 100755 index 1212d0470a2..00000000000 --- a/extras/lcov/lcov_prep +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e - -cd build-root -rm -rf html -lcov --capture --initial --directory . --output-file lcov.out diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fb4463e332e..b6d48f5b5ea 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -127,6 +127,7 @@ elseif (${CMAKE_BUILD_TYPE_LC} MATCHES "coverity") elseif (${CMAKE_BUILD_TYPE_LC} MATCHES "gcov") add_compile_options(-O0 -fprofile-arcs -ftest-coverage) add_compile_definitions(CLIB_DEBUG CLIB_GCOV) + link_libraries(gcov) endif() set(BUILD_TYPES release debug coverity gcov) diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c index 74ad19ebbc2..f09a537a48d 100644 --- a/src/vlib/unix/main.c +++ b/src/vlib/unix/main.c @@ -145,17 +145,6 @@ unix_signal_handler (int signum, siginfo_t * si, ucontext_t * uc) break; } -#ifdef CLIB_GCOV - /* - * Test framework sends SIGTERM, so we need to flush the - * code coverage stats here. - */ - { - void __gcov_flush (void); - __gcov_flush (); - } -#endif - /* Null terminate. */ vec_add1 (syslog_msg, 0); diff --git a/test/Makefile b/test/Makefile index 41d28f9f772..27e750137ef 100644 --- a/test/Makefile +++ b/test/Makefile @@ -80,7 +80,7 @@ PIP_TOOLS_VERSION=6.6.0 PIP_SETUPTOOLS_VERSION=62.1.0 PYTHON_DEPENDS=requirements-$(PYTHON_VERSION).txt SCAPY_SOURCE=$(shell find $(VENV_PATH)/lib/python* -name site-packages) -BUILD_COV_DIR=$(TEST_BR)/coverage +BUILD_COV_DIR=$(BR)/test-coverage PIP_TOOLS_INSTALL_DONE=$(VENV_RUN_DIR)/pip-tools-install-$(PYTHON_VERSION)-$(PIP_TOOLS_VERSION).done PIP_INSTALL_DONE=$(VENV_RUN_DIR)/pip-install-$(PYTHON_VERSION)-$(PIP_VERSION).done @@ -326,19 +326,36 @@ wipe: reset $(BUILD_COV_DIR): @mkdir -p $@ -.PHONY: cov -cov: wipe-cov test-dep ext $(BUILD_COV_DIR) +.PHONY: cov-prep +cov-prep: test-dep @lcov --zerocounters --directory $(VPP_BUILD_DIR) @test -z "$(EXTERN_COV_DIR)" || lcov --zerocounters --directory $(EXTERN_COV_DIR) - $(call retest-func) - @lcov --capture --directory $(VPP_BUILD_DIR) --output-file $(BUILD_COV_DIR)/coverage.info - @test -z "$(EXTERN_COV_DIR)" || lcov --capture --directory $(EXTERN_COV_DIR) --output-file $(BUILD_COV_DIR)/extern-coverage.info - @genhtml $(BUILD_COV_DIR)/coverage.info --output-directory $(BUILD_COV_DIR)/html - @test -z "$(EXTERN_COV_DIR)" || genhtml $(BUILD_COV_DIR)/extern-coverage.info --output-directory $(BUILD_COV_DIR)/extern-html + +.PHONY: cov-post +cov-post: wipe-cov $(BUILD_COV_DIR) + @lcov --capture \ + --directory $(VPP_BUILD_DIR) \ + --output-file $(BUILD_COV_DIR)/coverage.info + @test -z "$(EXTERN_COV_DIR)" || \ + lcov --capture \ + --directory $(EXTERN_COV_DIR) \ + --output-file $(BUILD_COV_DIR)/extern-coverage.info + @lcov --remove $(BUILD_COV_DIR)/coverage.info \ + "/usr/include/*" "*/build-root/*" "/opt/*" "/usr/lib/*" \ + -o $(BUILD_COV_DIR)/coverage-filtered.info + @genhtml $(BUILD_COV_DIR)/coverage-filtered.info \ + --output-directory $(BUILD_COV_DIR)/html + @test -z "$(EXTERN_COV_DIR)" || \ + genhtml $(BUILD_COV_DIR)/extern-coverage.info \ + --output-directory $(BUILD_COV_DIR)/extern-html @echo @echo "Build finished. Code coverage report is in $(BUILD_COV_DIR)/html/index.html" @test -z "$(EXTERN_COV_DIR)" || echo "Code coverage report for out-of-tree objects is in $(BUILD_COV_DIR)/extern-html/index.html" +.PHONY: cov +cov: + make -C . cov-prep test cov-post + .PHONY: wipe-cov wipe-cov: wipe @rm -rf $(BUILD_COV_DIR) @@ -388,14 +405,17 @@ help: @echo "" @echo " test - build and run (basic) functional tests" @echo " test-debug - build and run (basic) functional tests (debug build)" + @echo " test-cov - generate code coverage report for functional tests" + @echo " test-cov-prep - coverage phase #1 : prepare lcov" + @echo " test-cov-build - coverage phase #2 : build gcov image & run tests against it (use TEST=)" + @echo " test-cov-post - coverage phase #3 : generate lcov html report" @echo " test-all - build and run functional and extended tests" @echo " test-all-debug - build and run functional and extended tests (debug build)" + @echo " test-all-cov - generate code coverage report for functional and extended tests" @echo " retest - run functional tests" @echo " retest-debug - run functional tests (debug build)" @echo " retest-all - run functional and extended tests" @echo " retest-all-debug - run functional and extended tests (debug build)" - @echo " test-cov - generate code coverage report for test framework" - @echo " test-gcov - build and run functional tests (gcov build)" @echo " test-wipe - wipe (temporary) files generated by unit tests" @echo " test-wipe-cov - wipe code coverage report for test framework" @echo " test-wipe-papi - rebuild vpp_papi sources" @@ -556,6 +576,10 @@ help: @echo " enable debugging of the test framework itself (expert)" @echo " (default: no)" @echo "" + @echo " TEST_GCOV=[1|y|yes]" + @echo " enable tests specifically designed soley for code coverage" + @echo " (default: no)" + @echo "" @echo " API_FUZZ=[1|y|yes]" @echo " enable VPP api fuzz testing" @echo " (default: no)" @@ -569,7 +593,3 @@ help: @echo " test-start-vpp-in-gdb - start VPP in gdb (release)" @echo " test-start-vpp-debug-in-gdb - start VPP in gdb (debug)" @echo "" - @echo "Creating test code coverage report:" - @echo "" - @echo " test-cov - generate code coverage report for test framework" - @echo " test-wipe-cov - wipe code coverage report for test framework" |