summaryrefslogtreecommitdiffstats
path: root/.gitignore
blob: 64dbb8c144019aa354bca5f0a90d78b16694312e (plain) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

*~
*.sw[op]

/build-root/.ccache
/build-root/build-*/
/build-root/install-*/
/build-root/tools
/build-root/dpdk-includes/
/build-root/packages-vpp/
/build-root/path_setup
/build-root/deb/debian/vpp-plugins/
/build-root/deb/.pybuild/
/build-root/*.buildinfo
/build-root/*.deb
/build-root/*.rpm
/build-root/*.tar.xz
/build-root/*.changes
/build-root/rpmbuild/
/build-root/test-doc/
/build-root/test-cov/
/build-root/test/
/test/run/
/test/build/
/test/coverage/
/test/vapi_test/
/test/doc/build/
/build-config.mk
/build/external/*.tar.gz
/build/external/*.tar.xz
/build/external/vpp-*
/build/external/dpdk_mlx_default.sh
/build/external/downloads/
/path_setup
/tools/
# start autotools ignore
autom4te.cache/
config/
Makefile.in
aclocal.m4
app.info
compile
config.log
config.guess
config.sub
config.status
configure.scan
coverage_report
depcomp
install-sh
libtool
m4
missing
stamp-h1
ltmain.sh
ylwrap
test-driver

*.iml
.deps.ok
.settings
.autotools
# stop autotools ignore

# OSX and some IDE
.DS_Store
.idea/
.project
.cproject
.pydevproject
.vscode/
cmake-build*/

# cscope and ctags
/cscope.*
/tags
ID
TAGS
# ggtags
GPATH
GRTAGS
GTAGS
# Generated documentation
/build-root/docs
/build-root/.doxygen-bootstrap.ok
/build-root/.doxygen-siphon.dep
/docs/venv

# language servers
compile_commands.json
.clangd
.cache

# indent backup files
*.BAK

# Python bytecode
*.pyc

# Python api generator
/src/vpp-api/python/build
/src/vpp-api/python/dist
/src/vpp-api/python/vpp_papi.egg-info
/src/vpp_api/.tox
/src/vpp_api/.stestr

# vppctl command list
/src/scripts/vppctl-cmd-list

# No core files
**/core

# extra scripts config
/extras/scripts/.config/

# extras gomemif build files
/extras/gomemif/bazel*

# vpptop build files
/extras/vpptop/build/*

# debian packaging
.pc

# No test log files
**/test-run-*-*-*

# host stack test framework
/extras/hs-test/vpp-data
/extras/hs-test/hs-test
/extras/hs-test/http_server
/extras/hs-test/.build.vpp
66bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ } 
.PHONY: verify-python-path

VPP_TEST_FAILED_DIR=/tmp/vpp-failed-unittests/

verify-python-path:
ifndef VPP_PYTHON_PREFIX
	$(error VPP_PYTHON_PREFIX is not set)
endif

.PHONY: verify-no-running-vpp

ifdef VPP_ZOMBIE_NOCHECK
VPP_PIDS=
else
VPP_PIDS=$(shell pgrep -d, -x vpp_main)
endif

ifeq ($(DEBUG),gdb)
FORCE_FOREGROUND=1
else ifeq ($(DEBUG),gdbserver)
FORCE_FOREGROUND=1
else ifeq ($(DEBUG),core)
FORCE_FOREGROUND=1
else
FORCE_FOREGROUND=0
endif

ifdef PROFILE_OUTPUT
PROFILE_OUTPUT_OPTS=-o $(PROFILE_OUTPUT)
endif

ifndef PROFILE_SORT_BY
PROFILE_SORT_BY=cumtime
endif

ifeq ($(PROFILE),1)
PYTHON_PROFILE_OPTS=-m cProfile $(PROFILE_OUTPUT_OPTS) -s $(PROFILE_SORT_BY)
FORCE_FOREGROUND=1
endif

verify-no-running-vpp:
	@if [ "$(VPP_PIDS)" != "" ]; then \
		echo; \
		echo "*** Existing vpp processes detected (PID(s): $(VPP_PIDS)). Running tests under these conditions is not supported. ***"; \
		echo; \
		ps -fp $(VPP_PIDS);\
		echo; \
		false; \
	fi

UNITTEST_EXTRA_OPTS=
UNITTEST_FAILFAST_OPTS=

ifeq ($(FAILFAST),1)
UNITTEST_EXTRA_OPTS=-f
endif

ifneq ($(EXTERN_TESTS),)
UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS)
endif

ifeq ($(TEST_DEBUG),1)
VPP_PYTHON_PREFIX:=$(VPP_PYTHON_PREFIX)/debug
PYTHON_EXTRA_DEPENDS=objgraph pympler
else
PYTHON_EXTRA_DEPENDS=
endif

PYTHON_VENV_PATH=$(VPP_PYTHON_PREFIX)/virtualenv
PYTHON_DEPENDS=$(PYTHON_EXTRA_DEPENDS) faulthandler six scapy==2.4.0 pexpect cryptography subprocess32 cffi git+https://github.com/vpp-dev/py-lispnetworking
SCAPY_SOURCE=$(shell find $(PYTHON_VENV_PATH) -name site-packages)
BUILD_COV_DIR=$(BR)/test-cov

GET_PIP_SCRIPT=$(VPP_PYTHON_PREFIX)/get-pip.py
PIP_INSTALL_DONE=$(VPP_PYTHON_PREFIX)/pip-install.done
PIP_PATCH_DONE=$(VPP_PYTHON_PREFIX)/pip-patch.done
PAPI_INSTALL_DONE=$(VPP_PYTHON_PREFIX)/papi-install.done

PAPI_INSTALL_FLAGS=$(PIP_INSTALL_DONE) $(PIP_PATCH_DONE) $(PAPI_INSTALL_DONE)

ifeq ($(PYTHON),)
PYTHON_INTERP=python2.7
else
PYTHON_INTERP=$(PYTHON)
endif

$(GET_PIP_SCRIPT):
	@mkdir -p $(VPP_PYTHON_PREFIX)
	@bash -c "cd $(VPP_PYTHON_PREFIX) && curl -O https://bootstrap.pypa.io/get-pip.py"

$(PIP_INSTALL_DONE): $(GET_PIP_SCRIPT)
	@virtualenv $(PYTHON_VENV_PATH) -p $(PYTHON_INTERP)
	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python $(GET_PIP_SCRIPT)"
	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install $(PYTHON_DEPENDS)"
	@touch $@

$(PIP_PATCH_DONE): $(PIP_INSTALL_DONE)
	@echo --- patching ---
	@sleep 1 # Ensure python recompiles patched *.py files -> *.pyc
	for f in $(CURDIR)/patches/scapy-2.4/*.patch ; do \
		echo Applying patch: $$(basename $$f) ; \
		patch -p1 -d $(SCAPY_SOURCE) < $$f ; \
	done
	@touch $@

$(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE)
	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && cd $(WS_ROOT)/src/vpp-api/python && python setup.py install"
	@touch $@

define retest-func
@env FORCE_FOREGROUND=$(FORCE_FOREGROUND) VPP_TEST_FAILED_DIR=$(VPP_TEST_FAILED_DIR) scripts/setsid_wrapper.sh $(FORCE_FOREGROUND) $(PYTHON_VENV_PATH)/bin/activate python $(PYTHON_PROFILE_OPTS) run_tests.py -d $(TEST_DIR) $(UNITTEST_EXTRA_OPTS) || env VPP_TEST_FAILED_DIR=$(VPP_TEST_FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh
endef

.PHONY: sanity

ifeq ($(SANITY),no)
SANITY_IMPORT_VPP_PAPI_CMD=true
SANITY_RUN_VPP_CMD=true
else
SANITY_IMPORT_VPP_PAPI_CMD=source $(PYTHON_VENV_PATH)/bin/activate && python sanity_import_vpp_papi.py
SANITY_RUN_VPP_CMD=source $(PYTHON_VENV_PATH)/bin/activate && python sanity_run_vpp.py
endif

sanity: verify-no-running-vpp
	@sys_req/dev_shm_size.sh
	@bash -c "$(SANITY_IMPORT_VPP_PAPI_CMD) ||\
		(echo \"*******************************************************************\" &&\
		 echo \"* Sanity check failed, cannot import vpp_papi\" &&\
		 echo \"* to debug: \" &&\
		 echo \"* 1. enter test shell:   make test-shell\" &&\
		 echo \"* 2. execute debugger:   gdb python -ex 'run sanity_import_vpp_papi.py'\" &&\
	         echo \"*******************************************************************\" &&\
		 false)"
	@bash -c "$(SANITY_RUN_VPP_CMD) ||\
		(echo \"*******************************************************************\" &&\
		 echo \"* Sanity check failed, cannot run vpp\" &&\
	         echo \"*******************************************************************\" &&\
		 false)"

.PHONY: ext
ext:
	make -C ext

test-dep: verify-python-path $(PAPI_INSTALL_DONE)

test: verify-python-path $(PAPI_INSTALL_DONE) sanity reset
	$(call retest-func)

test-ext: verify-python-path $(PAPI_INSTALL_DONE) ext sanity reset
	$(call retest-func)

retest: verify-python-path sanity reset
	$(call retest-func)

shell: verify-python-path $(PAPI_INSTALL_DONE)
	@echo "source $(PYTHON_VENV_PATH)/bin/activate;\
		echo '***';\
		echo VPP_TEST_BUILD_DIR=$(VPP_TEST_BUILD_DIR);\
		echo VPP_TEST_BIN=$(VPP_TEST_BIN);\
		echo VPP_TEST_PLUGIN_PATH=$(VPP_TEST_PLUGIN_PATH);\
		echo VPP_TEST_INSTALL_PATH=$(VPP_TEST_INSTALL_PATH);\
		echo EXTERN_TESTS=$(EXTERN_TESTS);\
		echo EXTERN_PLUGINS=$(EXTERN_PLUGINS);\
                echo EXTERN_COV_DIR=$(EXTERN_COV_DIR);\
		echo LD_LIBRARY_PATH=$(LD_LIBRARY_PATH);\
		echo '***';\
		exec </dev/tty" | bash -i

.PHONY: wipe doc

reset:
	@rm -f /dev/shm/vpp-unittest-*
	@rm -rf /tmp/vpp-unittest-*
	@rm -rf $(VPP_TEST_FAILED_DIR)
	@mkdir $(VPP_TEST_FAILED_DIR)

wipe: reset
	@make -C ext clean
	@rm -rf $(PYTHON_VENV_PATH)
	@rm -f $(PAPI_INSTALL_FLAGS)

doc: verify-python-path $(PIP_PATCH_DONE)
	@virtualenv $(PYTHON_VENV_PATH) -p $(PYTHON_INTERP)
	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install sphinx sphinx-rtd-theme"
	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && make -C doc WS_ROOT=$(WS_ROOT) BR=$(BR) NO_VPP_PAPI=1 html"

.PHONY: wipe-doc

wipe-doc:
	@make -C doc wipe BR=$(BR)

cov: wipe-cov reset ext verify-python-path $(PAPI_INSTALL_DONE)
	@lcov --zerocounters --directory $(VPP_TEST_BUILD_DIR)
	@test -z "$(EXTERN_COV_DIR)" || lcov --zerocounters --directory $(EXTERN_COV_DIR)
	$(call retest-func)
	@mkdir $(BUILD_COV_DIR)
	@lcov --capture --directory $(VPP_TEST_BUILD_DIR) --output-file $(BUILD_COV_DIR)/coverage.info
	@test -z "$(EXTERN_COV_DIR)" || lcov --capture --directory $(EXTERN_COV_DIR) --output-file $(BUILD_COV_DIR)/extern-coverage.info
	@genhtml $(BUILD_COV_DIR)/coverage.info --output-directory $(BUILD_COV_DIR)/html
	@test -z "$(EXTERN_COV_DIR)" || genhtml $(BUILD_COV_DIR)/extern-coverage.info --output-directory $(BUILD_COV_DIR)/extern-html
	@echo
	@echo "Build finished. Code coverage report is in $(BUILD_COV_DIR)/html/index.html"
	@test -z "$(EXTERN_COV_DIR)" || echo "Code coverage report for out-of-tree objects is in $(BUILD_COV_DIR)/extern-html/index.html"

.PHONY: wipe-cov

wipe-cov: wipe
	@rm -rf $(BUILD_COV_DIR)

.PHONY: checkstyle
checkstyle: verify-python-path
	@virtualenv $(PYTHON_VENV_PATH) -p $(PYTHON_INTERP)
	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install pycodestyle"
	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate &&\
		pycodestyle --show-source --ignore=W504,E126,E241,E226,E305,E704,E741,E722 -v $(WS_ROOT)/test/*.py ||\
		(echo \"*******************************************************************\" &&\
		 echo \"* Test framework PEP8 compliance check FAILED \" &&\
	         echo \"*******************************************************************\" &&\
		 false)"
	@echo "*******************************************************************"
	@echo "* Test framework PEP8 compliance check passed"
	@echo "*******************************************************************"

help:
	@echo "Running tests:"
	@echo ""
	@echo " test                - build and run (basic) functional tests"
	@echo " test-debug          - build and run (basic) functional tests (debug build)"
	@echo " test-all            - build and run (all) functional tests"
	@echo " test-all-debug      - build and run (all) functional tests (debug build)"
	@echo " retest              - run functional tests"
	@echo " retest-debug        - run functional tests (debug build)"
	@echo " test-wipe           - wipe (temporary) files generated by unit tests"
	@echo " test-shell          - enter shell with test environment"
	@echo " test-shell-debug    - enter shell with test environment (debug build)"
	@echo ""
	@echo "Arguments controlling test runs:"
	@echo " V=[0|1|2]            - set test verbosity level"
	@echo " CACHE_OUTPUT=[0|1]   - cache VPP stdout/stderr and log as one block after test finishes (default: 1)"
	@echo " FAILFAST=[0|1]       - fail fast if 1, complete all tests if 0"
	@echo " TIMEOUT=<timeout>    - fail test suite if any single test takes longer than <timeout> (in seconds) to finish"
	@echo " RETRIES=<n>          - retry failed tests <n> times"
	@echo " DEBUG=<type>         - set VPP debugging kind"
	@echo "    DEBUG=core        - detect coredump and load it in gdb on crash"
	@echo "    DEBUG=gdb         - allow easy debugging by printing VPP PID "
	@echo "                        and waiting for user input before running "
	@echo "                        and tearing down a testcase"
	@echo "    DEBUG=gdbserver   - run gdb inside a gdb server, otherwise "
	@echo "                        same as above"
	@echo " STEP=[yes|no]        - ease debugging by stepping through a testcase "
	@echo " SANITY=[yes|no]      - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)"
	@echo " EXTENDED_TESTS=[1|y] - run extended tests"
	@echo " TEST=<filter>        - filter the set of tests:"
	@echo "    by file-name      - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py"
	@echo "    by file-suffix    - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py"
	@echo "    by wildcard       - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'"
	@echo "                        e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name"
	@echo "                             TEST='bfd.*.*' is equivalent to above example of filter by file-suffix"
	@echo "                             TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class"
	@echo "                             TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase"
	@echo "                             TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes"
	@echo ""
	@echo " VPP_ZOMBIE_NOCHECK=1 - skip checking for vpp (zombie) processes (CAUTION)"
	@echo " COREDUMP_SIZE=<size> - pass <size> as unix { coredump-size <size> } argument to vpp"
	@echo "                        e.g. COREDUMP_SIZE=4g"
	@echo "                             COREDUMP_SIZE=unlimited"
	@echo " EXTERN_TESTS=<path>  - path to out-of-tree test_<name>.py files containing test cases"
	@echo " EXTERN_PLUGINS=<path>- path to out-of-tree plugins to be loaded by vpp under test"
	@echo " EXTERN_COV_DIR=<path>- path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report"
	@echo ""
	@echo " PROFILE=1            - enable profiling of test framework via cProfile module"
	@echo " PROFILE_SORT_BY=opt  - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)"
	@echo " PROFILE_OUTPUT=file  - output profiling info to file - use absolute path (default: stdout)"
	@echo ""
	@echo " TEST_DEBUG=1         - turn on debugging of the test framework itself (expert)"
	@echo ""
	@echo "Creating test documentation"
	@echo " test-doc            - generate documentation for test framework"
	@echo " test-wipe-doc       - wipe documentation for test framework"
	@echo ""
	@echo "Creating test code coverage report"
	@echo " test-cov            - generate code coverage report for test framework"
	@echo " test-wipe-cov       - wipe code coverage report for test framework"
	@echo ""
	@echo "Verifying code-style"
	@echo " test-checkstyle     - check PEP8 compliance"
	@echo ""