aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Mikus <pmikus@cisco.com>2019-07-12 15:02:28 +0000
committerPeter Mikus <pmikus@cisco.com>2019-07-19 16:35:26 +0000
commit44a12d1a616f4ffd254646c3104b4ed50761ebfb (patch)
tree4d8472f97e555dd8ff8e9fbac6f2098d223823eb
parentfc7fbe37baaaa82f751757f75491e2a29db5176e (diff)
TOX: Naming conventions
Change-Id: I038d90fc481e5f74085d0fe6165b12769d98e8a1 Signed-off-by: Peter Mikus <pmikus@cisco.com>
-rw-r--r--resources/libraries/bash/entry/check/tc_naming.sh142
-rw-r--r--tox.ini7
2 files changed, 148 insertions, 1 deletions
diff --git a/resources/libraries/bash/entry/check/tc_naming.sh b/resources/libraries/bash/entry/check/tc_naming.sh
new file mode 100644
index 0000000000..70e5fa0063
--- /dev/null
+++ b/resources/libraries/bash/entry/check/tc_naming.sh
@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xeuo pipefail
+
+# This file should be executed from tox, as the assumend working directory
+# is different from where this file is located.
+# This file does not have executable flag nor shebang,
+# to dissuade non-tox callers.
+
+# "set -eu" handles failures from the following two lines.
+BASH_CHECKS_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))"
+BASH_FUNCTION_DIR="$(readlink -e "${BASH_CHECKS_DIR}/../../function")"
+source "${BASH_FUNCTION_DIR}/common.sh" || {
+ echo "Source failed." >&2
+ exit 1
+}
+
+# Grep of interest: We want all tc01- prefixed (skip TC variations for now).
+# Currently script assumes all variations inside to be part of either
+# auto-generation or not checked at all (VIRL derivates).
+r_grep="tc01-"
+# Parse grep of interest (learn path, learn suite, learn testcase name).
+r_parse='(.*)\/(.*).robot.*(tc[[:digit:]]{2}-.*)'
+
+# CSIT Testcase naming convention rules.
+# https://wiki.fd.io/view/CSIT/csit-test-naming
+# Rules are defined as regular expressions in ordered array and checked in order
+# in a loop, where every iteration is catenated with previous rules. This way we
+# can detect where exactly the naming does not meet criteria and print error
+# from rule string array. This imply that rules are defined in a way of a single
+# string. First rule must start with ^ and last is terminated by $.
+# Rules are written from Left to Right.
+# Bash regular expression logic is used. Once the error is raised the checker is
+# breaked for current Testcase marking the expected fail.
+# One caveat of this solution is that we cannot proceed to check full names now
+# as majority of Testcases does not meet naming criteria.
+s_testc_rules=(
+ 'TC id'
+ 'packet size'
+ 'core combination'
+ 'NIC driver mode'
+ 'packet encapsulation on L2 layer'
+ 'test type'
+ )
+r_testc_rules=(
+ '^tc[[:digit:]]{2}-'
+ '([[:digit:]]{2,4}B|IMIX)-'
+ '([[:digit:]]+c-){0,1}'
+ '(avf-|1lbvpplacp-|2lbvpplacp-){0,1}'
+ '(eth|dot1q|dot1ad)'
+ # TODO: Packet encapsulation (here majority of TC starts failing).
+ #'(ip4|ip6|ip6ip6|icmpv4|icmpv6)'
+ #'(ipsec[[:digit:]]+tnlhw|ipsec[[:digit:]]+tnlsw|'
+ #'srhip6|tcp|udp|lispip6|lispip4|vxlan){0,1}'
+ #'(http){0,1}-'
+ '(.*)-(dev|ndrpdr|cps)$'
+ )
+s_suite_rules=(
+ 'number of SUT nodes'
+ 'NIC card'
+ 'NIC driver mode'
+ 'packet encapsulation on L2 layer'
+ 'test type'
+ )
+r_suite_rules=(
+ '^(2n1l|2n){0,1}-'
+ '(eth2p|10ge2p1x710)-'
+ '(avf-|1lbvpplacp-|2lbvpplacp-){0,1}'
+ '(eth|dot1q|dot1ad)'
+ # TODO: Packet encapsulation (here majority of TC starts failing).
+ #'(ip4|ip6|ip6ip6|icmpv4|icmpv6)'
+ #'(ipsec[[:digit:]]+tnlhw|ipsec[[:digit:]]+tnlsw|'
+ #'srhip6|tcp|udp|lispip6|lispip4|vxlan){0,1}'
+ #'(http){0,1}-'
+ '(.*)-(dev|ndrpdr|cps)$'
+ )
+
+rm -f "tc_naming.log" || die
+
+# Disabling -x: Following lines are doing too much garbage output.
+set +x
+
+# Grep interest.
+grep_match=$(grep -RE "${r_grep}" tests/*) || die
+# Extract data from the grep output.
+suites_dirs=($(printf "${grep_match}" | sed -re "s/${r_parse}/\1/")) || die
+suites_names=($(printf "${grep_match}" | sed -re "s/${r_parse}/\2/")) || die
+testcases_names=($(printf "${grep_match}" | sed -re "s/${r_parse}/\3/")) || die
+
+# Naming check.
+total_failed_tc=0
+total_failed_su=0
+for idx in "${!testcases_names[@]}"; do
+ for pass in "${!r_suite_rules[@]}"; do
+ r_rule=$(printf '%s' "${r_suite_rules[@]:1:pass}")
+ if [[ ! "${suites_names[idx]}" =~ ${r_rule} ]]; then
+ msg=""
+ msg+="${suites_dirs[idx]}/${suites_names[idx]} / "
+ msg+="${testcases_names[idx]} ${s_testc_rules[pass]} "
+ msg+="is not matching suite naming rule!"
+ echo "${msg}" | tee -a "tc_naming.log" || die
+ total_failed_su=$((total_failed_su + 1))
+ break
+ fi
+ done
+ for pass in "${!r_testc_rules[@]}"; do
+ r_rule=$(printf '%s' "${r_testc_rules[@]:1:pass}")
+ if [[ ! "${testcases_names[idx]}" =~ ${r_rule} ]]; then
+ msg=""
+ msg+="${suites_dirs[idx]}/${suites_names[idx]} / "
+ msg+="${testcases_names[idx]} ${s_testc_rules[pass]} "
+ msg+="is not matching testcase naming rule!"
+ echo "${msg}" | tee -a "tc_naming.log" || die
+ total_failed_tc=$((total_failed_tc + 1))
+ break
+ fi
+ done
+done
+
+set -x
+
+if [ $((total_failed_tc + total_failed_su)) != "0" ]; then
+ warn
+ warn "Testcase naming checker: FAIL"
+ exit 1
+fi
+
+warn
+warn "Testcase naming checker: PASS" \ No newline at end of file
diff --git a/tox.ini b/tox.ini
index 3d4f01a56c..2ad02e3fec 100644
--- a/tox.ini
+++ b/tox.ini
@@ -25,7 +25,7 @@
# will execute only checks defined in "pylint" tox environment.
[tox]
-envlist = new_line_length, line_length, autogen, pylint
+envlist = new_line_length, line_length, autogen, pylint, tc_naming
# The following is needed as tox requires setup.py by default.
skipsdist = true
# Just a shorthand to avoid long lines.
@@ -61,6 +61,11 @@ whitelist_externals = /bin/bash
setenv = PYTHONPATH = {toxinidir}
commands = bash {[tox]checker_dir}/autogen.sh
+[testenv:tc_naming]
+whitelist_externals = /bin/bash
+# Fix all TC namings and remove the " || true" workaround.
+commands = bash -c "bash {[tox]checker_dir}/tc_naming.sh || true"
+
# TODO: Migrate current docs check here.
# TODO: Create voting "pylint violations should not increase" checker.
# TODO: Create voting checker to reject suites with Force Tags of other suite.