diff options
| -rw-r--r-- | resources/libraries/bash/entry/check/tc_coverage.sh | 100 | ||||
| -rw-r--r-- | tox.ini | 7 |
2 files changed, 106 insertions, 1 deletions
diff --git a/resources/libraries/bash/entry/check/tc_coverage.sh b/resources/libraries/bash/entry/check/tc_coverage.sh new file mode 100644 index 0000000000..2fda5f7d52 --- /dev/null +++ b/resources/libraries/bash/entry/check/tc_coverage.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# Copyright (c) 2019 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xeuo pipefail + +# This file should be executed from tox, as the assumend working directory +# is different from where this file is located. +# This file does not have executable flag nor shebang, +# to dissuade non-tox callers. + +# "set -eu" handles failures from the following two lines. +BASH_CHECKS_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" +BASH_FUNCTION_DIR="$(readlink -e "${BASH_CHECKS_DIR}/../../function")" +source "${BASH_FUNCTION_DIR}/common.sh" || { + echo "Source failed." >&2 + exit 1 +} + +# Grep of interest: We want all tc01- prefixed (skip TC variations for now). +# Currently script assumes all variations inside to be part of either +# auto-generation or not checked at all (VIRL derivates). +r_grep="tc01-" +# Parse grep of interest (learn path, learn suite, learn testcase name). +r_parse='(.*)\/(.*).robot.*(tc[[:digit:]]{2}-.*)' + +rm -f "tc_coverage.log" || die + +# Disabling -x: Following lines are doing too much garbage output. +set +x + +# Grep interest. +grep_match=$(grep -RE "${r_grep}" tests/*) || die +# Extract data from the grep output. +suites_dirs=($(printf "${grep_match}" | sed -re "s/${r_parse}/\1/")) || die +suites_names=($(printf "${grep_match}" | sed -re "s/${r_parse}/\2/")) || die +testcases_names=($(printf "${grep_match}" | sed -re "s/${r_parse}/\3/")) || die + +# Extract 2N suites from the global testcase list and normalize output. +suites_2n=($(printf '%s\n' "${suites_names[@]}" | \ + grep -E "^(2n1l|2n)-" | \ + sed -re "s/(2n1l|2n)-//")) +# Extract 3N suites from the global testcase list. +suites_3n=($(printf '%s\n' "${suites_names[@]}" | \ + grep -vE "^(2n1l|2n|eth2p)-")) +# Extract vpp_device suites from the global testcase list. +# Code will try to map the naming to perf by normalizing output. +suites_dev=($(printf '%s\n' "${suites_names[@]}" | \ + grep -E "^eth2p-" | \ + sed -re "s/eth2p-/10ge2p1x710-/" | \ + sed -re "s/-dev/-ndrpdr/" | \ + sed -re "s/-ethicmpv/-ethip/")) + +# Compute intersection of arrays. +intersection_3n2n=($(comm -12 <(printf '%s\n' "${suites_3n[@]}" | sort) \ + <(printf '%s\n' "${suites_2n[@]}" | sort))) +intersection_3ndev=($(comm -12 <(printf '%s\n' "${suites_2n[@]}" | sort) \ + <(printf '%s\n' "${suites_dev[@]}" | sort))) + +# Print the results in CSV format. +echo "Suite name;Testcase name;2n version;VPP Device version" \ + > tc_coverage.log +for i in "${!suites_names[@]}"; do + if [[ ! "${suites_names[i]}" =~ ^(2n1l|2n|eth2p)-.* ]]; then + echo -n "${suites_names[i]};${testcases_names[i]}" \ + >> tc_coverage.log + if [[ "${intersection_3n2n[@]}" =~ "${suites_names[i]}" ]]; then + echo -n ";yes" >> tc_coverage.log + else + echo -n ";no" >> tc_coverage.log + fi + if [[ "${intersection_3ndev[@]}" =~ "${suites_names[i]}" ]]; then + echo ";yes" >> tc_coverage.log + else + echo ";no" >> tc_coverage.log + fi + fi +done + +set -x + +echo "Count 2n: ${#suites_2n[@]}" +echo "Count 3n: ${#suites_3n[@]}" +echo "Count dev: ${#suites_dev[@]}" +echo "Coverage 2n3n: ${#intersection_3n2n[@]}" +echo "Coverage 3ndev: ${#intersection_3ndev[@]}" + +warn +warn "Testcase coverage checker: PASS" @@ -25,7 +25,7 @@ # will execute only checks defined in "pylint" tox environment. [tox] -envlist = new_line_length, line_length, autogen, pylint, tc_naming +envlist = new_line_length, line_length, autogen, pylint, tc_naming, tc_coverage # The following is needed as tox requires setup.py by default. skipsdist = true # Just a shorthand to avoid long lines. @@ -66,6 +66,11 @@ whitelist_externals = /bin/bash # Fix all TC namings and remove the " || true" workaround. commands = bash -c "bash {[tox]checker_dir}/tc_naming.sh || true" +[testenv:tc_coverage] +whitelist_externals = /bin/bash +# Coverage is not needed to be voting. +commands = bash -c "bash {[tox]checker_dir}/tc_coverage.sh || true" + # TODO: Migrate current docs check here. # TODO: Create voting "pylint violations should not increase" checker. # TODO: Create voting checker to reject suites with Force Tags of other suite. |