CSIT-1135: Scripts for VPP per-patch testing

+ Scripts do not rely on (other) bootstraps.
+ Perf verify bootstrap is also the new style sript now.
+ Scripts are divided to functions for better re-use.
+ Functions are sourced from small number of large "library" files.
- Still using jumpavg from pipy.
- Perpatch has specific simplified parsing (instead of the PAL one).
- Bash style document is in a separate Change.

Change-Id: If88fa528ce155ea86b614e3d77c0550b91bbdf11
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

1 files changed, 4 insertions, 396 deletions

diff --git a/bootstrap-verify-perf.sh b/bootstrap-verify-perf.sh
index c72d6ec7dd..afcfd1eca9 100755
--- a/bootstrap-verify-perf.sh
+++ b/bootstrap-verify-perf.sh
@@ -12,400 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -xo pipefail
+set -exuo pipefail
 
-# FUNCTIONS
-function warn () {
-    # Prints the message to standard error.
-    echo "$@" >&2
-}
-
-function die () {
-    # Prints the message to standard error end exit with error code specified
-    # by first argument.
-    status="$1"
-    shift
-    warn "$@"
-    exit "$status"
-}
-
-function help () {
-    # Displays help message.
-    die 1 "Usage: `basename $0` csit-[dpdk|vpp|ligato]-[2n-skx|3n-skx|3n-hsw]"
-}
-
-function cancel_all () {
-    # Trap function to get into consistent state.
-    python ${SCRIPT_DIR}/resources/tools/scripts/topo_cleanup.py -t $1 || {
-        die 1 "Failure during execution of topology cleanup script!"
-    }
-    python ${SCRIPT_DIR}/resources/tools/scripts/topo_reservation.py -c -t $1 || {
-        die 1 "Failure during execution of topology un-reservation script!"
-    }
-}
-
-# VARIABLES
-# Space separated list of available testbeds, described by topology files
-TOPOLOGIES_3N_HSW=(topologies/available/lf_3n_hsw_testbed1.yaml
-                   topologies/available/lf_3n_hsw_testbed2.yaml
-                   topologies/available/lf_3n_hsw_testbed3.yaml)
-TOPOLOGIES_2N_SKX=(topologies/available/lf_2n_skx_testbed21.yaml
-                   topologies/available/lf_2n_skx_testbed24.yaml)
-TOPOLOGIES_3N_SKX=(topologies/available/lf_3n_skx_testbed31.yaml
-                   topologies/available/lf_3n_skx_testbed32.yaml)
-
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-export PYTHONPATH=${SCRIPT_DIR}
-
-RESERVATION_DIR="/tmp/reservation_dir"
-DOWNLOAD_DIR="${SCRIPT_DIR}/download_dir"
-ARCHIVE_DIR="${SCRIPT_DIR}/archive"
-
-mkdir -p ${DOWNLOAD_DIR} || {
-    die 1 "Failed to create download dir!"
-}
-mkdir -p ${ARCHIVE_DIR} || {
-    die 1 "Failed to create archive dir!"
-}
-
-# Get test code.
-TEST_CODE=${JOB_NAME-}
-if [[ -z ${TEST_CODE} ]]; then
-    TEST_CODE=${1}
-    shift
-fi
-
-# TOPOLOGY SELECTION
-case "$TEST_CODE" in
-    *2n-skx*)
-        TOPOLOGIES=${TOPOLOGIES_2N_SKX[@]}
-        TOPOLOGIES_TAGS="2_node_*_link_topo"
-        ;;
-    *3n-skx*)
-        TOPOLOGIES=${TOPOLOGIES_3N_SKX[@]}
-        TOPOLOGIES_TAGS="3_node_*_link_topo"
-        ;;
-    *)
-        # Fallback to 3-node Haswell by default (backward compatibility)
-        TOPOLOGIES=${TOPOLOGIES_3N_HSW[@]}
-        TOPOLOGIES_TAGS="3_node_*_link_topo"
-        ;;
-esac
-
-if [[ -z "${TOPOLOGIES}" ]]; then
-    die 1 "No applicable topology found!"
-fi
-
-cd ${DOWNLOAD_DIR}
-case "$TEST_CODE" in
-    *hc2vpp*)
-        DUT="hc2vpp"
-        ;;
-    *vpp*)
-        DUT="vpp"
-
-        case "$TEST_CODE" in
-            csit-vpp-*)
-                # Use downloaded packages with specific version
-                if [[ "$TEST_CODE" == *daily* ]] || \
-                   [[ "$TEST_CODE" == *weekly* ]] || \
-                   [[ "$TEST_CODE" == *timed* ]];
-                then
-                    echo Downloading latest VPP packages from NEXUS...
-                    bash ${SCRIPT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh \
-                        --skip-install || {
-                        die 1 "Failed to get VPP packages!"
-                    }
-                else
-                    echo Downloading VPP packages of specific version from NEXUS...
-                    DPDK_STABLE_VER=$(cat ${SCRIPT_DIR}/DPDK_STABLE_VER)
-                    VPP_STABLE_VER=$(cat ${SCRIPT_DIR}/VPP_STABLE_VER_UBUNTU)
-                    bash ${SCRIPT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh \
-                        --skip-install --vpp ${VPP_STABLE_VER} --dkms ${DPDK_STABLE_VER} || {
-                        die 1 "Failed to get VPP packages!"
-                    }
-                fi
-                ;;
-            vpp-csit-*)
-                # Use local built packages.
-                mv ../${DUT}*.deb ${DOWNLOAD_DIR}/
-                ;;
-            *)
-                die 1 "Unable to identify job type from: ${TEST_CODE}!"
-                ;;
-        esac
-        ;;
-    *ligato*)
-        DUT="kubernetes"
-
-        case "$TEST_CODE" in
-            csit-*)
-                # Use downloaded packages with specific version
-                if [[ "$TEST_CODE" == *daily* ]] || \
-                   [[ "$TEST_CODE" == *weekly* ]] || \
-                   [[ "$TEST_CODE" == *timed* ]];
-                then
-                    echo Downloading latest VPP packages from NEXUS...
-                    bash ${SCRIPT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh \
-                        --skip-install || {
-                        die 1 "Failed to get VPP packages!"
-                    }
-                else
-                    echo Downloading VPP packages of specific version from NEXUS...
-                    DPDK_STABLE_VER=$(cat ${SCRIPT_DIR}/DPDK_STABLE_VER)
-                    VPP_STABLE_VER=$(cat ${SCRIPT_DIR}/VPP_STABLE_VER_UBUNTU)
-                    bash ${SCRIPT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh \
-                        --skip-install --vpp ${VPP_STABLE_VER} --dkms ${DPDK_STABLE_VER} || {
-                        die 1 "Failed to get VPP packages!"
-                    }
-                fi
-                ;;
-            vpp-csit-*)
-                # Use local builded packages.
-                mv ../${DUT}*.deb ${DOWNLOAD_DIR}/
-                ;;
-            *)
-                die 1 "Unable to identify job type from: ${TEST_CODE}!"
-                ;;
-        esac
-        # Extract VPP API to specific folder
-        dpkg -x ${DOWNLOAD_DIR}/vpp_*.deb /tmp/vpp || {
-            die 1 "Failed to extract ${DUT} package!"
-        }
-
-        LIGATO_REPO_URL="https://github.com/ligato/"
-        VPP_AGENT_STABLE_VER=$(cat ${SCRIPT_DIR}/VPP_AGENT_STABLE_VER)
-        DOCKER_DEB="docker-ce_18.03.0~ce-0~ubuntu_amd64.deb"
-
-        # Clone & checkout stable vnf-agent
-        cd ../..
-        git clone -b ${VPP_AGENT_STABLE_VER} --single-branch \
-            ${LIGATO_REPO_URL}/vpp-agent vpp-agent || {
-            die 1 "Failed to run: git clone ${LIGATO_REPO_URL}/vpp-agent!"
-        }
-        cd vpp-agent
-
-        # Install Docker
-        wget -q https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/${DOCKER_DEB} || {
-            die 1 "Failed to download Docker package!"
-        }
-
-        sudo dpkg -i ${DOCKER_DEB} || {
-            die 1 "Failed to install Docker!"
-        }
-
-        # Pull ligato/dev_vpp_agent docker image and re-tag as local
-        sudo docker pull ligato/dev-vpp-agent:${VPP_AGENT_STABLE_VER} || {
-            die 1 "Failed to pull Docker image!"
-        }
-
-        sudo docker tag ligato/dev-vpp-agent:${VPP_AGENT_STABLE_VER}\
-            dev_vpp_agent:latest || {
-            die 1 "Failed to tag Docker image!"
-        }
-
-        # Start dev_vpp_agent container as daemon
-        sudo docker run --rm -itd --name agentcnt dev_vpp_agent bash || {
-            die 1 "Failed to run Docker image!"
-        }
-
-        # Copy latest vpp api into running container
-        sudo docker cp /tmp/vpp/usr/share/vpp/api agentcnt:/usr/share/vpp || {
-            die 1 "Failed to copy files Docker image!"
-        }
-
-        for f in ${DOWNLOAD_DIR}/*; do
-            sudo docker cp $f agentcnt:/opt/vpp-agent/dev/vpp/build-root/ || {
-                die 1 "Failed to copy files Docker image!"
-            }
-        done
-
-        # Recompile vpp-agent
-        sudo docker exec -i agentcnt \
-            script -qec '. ~/.bashrc; cd /go/src/github.com/ligato/vpp-agent && make generate && make install' || {
-            die 1 "Failed to build vpp-agent in Docker image!"
-        }
-        # Save container state
-        sudo docker commit `sudo docker ps -q` dev_vpp_agent:latest || {
-            die 1 "Failed to commit state of Docker image!"
-        }
-
-        # Build prod_vpp_agent docker image
-        cd docker/prod/ &&\
-            sudo docker build --tag prod_vpp_agent --no-cache . || {
-                die 1 "Failed to build Docker image!"
-            }
-        # Export Docker image
-        sudo docker save prod_vpp_agent | gzip > prod_vpp_agent.tar.gz || {
-            die 1 "Failed to save Docker image!"
-        }
-        DOCKER_IMAGE="$( readlink -f prod_vpp_agent.tar.gz | tr '\n' ' ' )"
-        rm -r ${DOWNLOAD_DIR}/vpp*
-        mv ${DOCKER_IMAGE} ${DOWNLOAD_DIR}/
-        ;;
-    *dpdk*)
-        DUT="dpdk"
-
-        DPDK_REPO='https://fast.dpdk.org/rel/'
-        # Use downloaded packages with specific version
-        if [[ "$TEST_CODE" == *daily* ]] || \
-           [[ "$TEST_CODE" == *weekly* ]] || \
-           [[ "$TEST_CODE" == *timed* ]];
-        then
-            echo "Downloading latest DPDK packages from repo..."
-            DPDK_STABLE_VER=$(wget --no-check-certificate --quiet -O - ${DPDK_REPO} | \
-                grep -v '2015' | grep -Eo 'dpdk-[^\"]+xz' | tail -1)
-        else
-            echo "Downloading DPDK packages of specific version from repo..."
-            DPDK_STABLE_VER='dpdk-18.05.tar.xz'
-        fi
-        if [[ ! -f ${DPDK_STABLE_VER} ]]; then
-            wget --no-check-certificate ${DPDK_REPO}${DPDK_STABLE_VER} || {
-                die 1 "Failed to get DPDK package from ${DPDK_REPO}!"
-            }
-        fi
-        ;;
-    *)
-        die 1 "Unable to identify DUT type from: ${TEST_CODE}!"
-        ;;
-esac
-cd ${SCRIPT_DIR}
-
-if [[ ! "$(ls -A ${DOWNLOAD_DIR})" ]]; then
-    die 1 "No artifacts downloaded!"
-fi
-
-# ENVIRONMENT PREPARATION
-rm -rf env
-
-pip install virtualenv || {
-    die 1 "Failed to install virtual env!"
-}
-virtualenv --system-site-packages env || {
-    die 1 "Failed to create virtual env!"
-}
-source env/bin/activate || {
-    die 1 "Failed to activate virtual env!"
-}
-pip install -r requirements.txt || {
-    die 1 "Failed to install requirements to virtual env!"
-}
-
-# We iterate over available topologies and wait until we reserve topology.
-while :; do
-    for TOPOLOGY in ${TOPOLOGIES};
-    do
-        python ${SCRIPT_DIR}/resources/tools/scripts/topo_reservation.py -t ${TOPOLOGY}
-        if [ $? -eq 0 ]; then
-            WORKING_TOPOLOGY=${TOPOLOGY}
-            echo "Reserved: ${WORKING_TOPOLOGY}"
-            # On script exit we clean testbed.
-            trap "cancel_all ${WORKING_TOPOLOGY}" EXIT
-            break
-        fi
-    done
-
-    if [ -n "${WORKING_TOPOLOGY}" ]; then
-        # Exit the infinite while loop if we made a reservation.
-        break
-    fi
-
-    # Wait ~3minutes before next try.
-    SLEEP_TIME=$[ ( $RANDOM % 20 ) + 180 ]s
-    echo "Sleeping ${SLEEP_TIME}"
-    sleep ${SLEEP_TIME}
-done
-
-# Clean testbed before execution.
-python ${SCRIPT_DIR}/resources/tools/scripts/topo_cleanup.py -t ${WORKING_TOPOLOGY} || {
-    die 1 "Failed to cleanup topologies!"
-}
-
-# CSIT EXECUTION
-PYBOT_ARGS="--outputdir ${ARCHIVE_DIR} --loglevel TRACE --variable TOPOLOGY_PATH:${WORKING_TOPOLOGY} --suite tests.${DUT}.perf"
-
-# NIC SELECTION
-# All topologies NICs
-TOPOLOGIES_NICS=($(grep -hoPR "model: \K.*" topologies/available/* | sort -u))
-# Selected topology NICs
-TOPOLOGY_NICS=($(grep -hoPR "model: \K.*" ${WORKING_TOPOLOGY} | sort -u))
-# All topologies NICs - Selected topology NICs
-EXCLUDE_NICS=($(comm -13 <(printf '%s\n' "${TOPOLOGY_NICS[@]}") <(printf '%s\n' "${TOPOLOGIES_NICS[@]}")))
-
-case "$TEST_CODE" in
-    # Select specific performance tests based on jenkins job type variable.
-    *ndrpdr-weekly* )
-        TEST_TAG_ARRAY=(ndrpdrAND64bAND1c
-                        ndrpdrAND78bAND1c)
-        ;;
-    *mrr-daily* | *mrr-weekly* )
-        TEST_TAG_ARRAY=(mrrAND64bAND1c
-                        mrrAND64bAND2c
-                        mrrAND64bAND4c
-                        mrrAND78bAND1c
-                        mrrAND78bAND2c
-                        mrrAND78bAND4c
-                        mrrANDimixAND1cANDvhost
-                        mrrANDimixAND2cANDvhost
-                        mrrANDimixAND4cANDvhost
-                        mrrANDimixAND1cANDmemif
-                        mrrANDimixAND2cANDmemif
-                        mrrANDimixAND4cANDmemif)
-        ;;
-    * )
-        if [[ -z "$TEST_TAG_STRING" ]]; then
-            # If nothing is specified, we will run pre-selected tests by
-            # following tags. Items of array will be concatenated by OR in Robot
-            # Framework.
-            TEST_TAG_ARRAY=(mrrANDnic_intel-x710AND1cAND64bANDip4base
-                            mrrANDnic_intel-x710AND1cAND78bANDip6base
-                            mrrANDnic_intel-x710AND1cAND64bANDl2bdbase)
-        else
-            # If trigger contains tags, split them into array.
-            TEST_TAG_ARRAY=(${TEST_TAG_STRING//:/ })
-        fi
-        ;;
-esac
-
-# We will add excluded NICs.
-TEST_TAG_ARRAY+=("${EXCLUDE_NICS[@]/#/!NIC_}")
-
-TAGS=()
-
-# We will prefix with perftest to prevent running other tests (e.g. Functional).
-prefix="perftestAND"
-if [[ ${TEST_CODE} == vpp-* ]]; then
-    # Automatic prefixing for VPP jobs to limit the NIC used and
-    # traffic evaluation to MRR.
-    prefix="${prefix}mrrANDnic_intel-x710AND"
-fi
-for TAG in "${TEST_TAG_ARRAY[@]}"; do
-    if [[ ${TAG} == "!"* ]]; then
-        # Exclude tags are not prefixed.
-        TAGS+=("${TAG}")
-    else
-        TAGS+=("$prefix${TAG}")
-    fi
-done
-
-# Catenate TAG selections
-EXPANDED_TAGS=()
-for TAG in "${TAGS[@]}"; do
-    if [[ ${TAG} == "!"* ]]; then
-        EXPANDED_TAGS+=(" --exclude ${TAG#$"!"} ")
-    else
-        EXPANDED_TAGS+=(" --include ${TOPOLOGIES_TAGS}AND${TAG} ")
-    fi
-done
-
-# Execute the test
-pybot ${PYBOT_ARGS}${EXPANDED_TAGS[@]} tests/
-RETURN_STATUS=$(echo $?)
-
-# We will create additional archive if workspace variable is set. This way if
-# script is running in jenkins all will be automatically archived to logs.fd.io.
-if [[ -n ${WORKSPACE-} ]]; then
-    cp -r ${ARCHIVE_DIR}/ $WORKSPACE/archives/
-fi
-
-exit ${RETURN_STATUS}
+# TODO: Delete this file, perhaps replacing it with a symlink.
+here=$(dirname $(readlink -e "${BASH_SOURCE[0]}"))
+source "${here}/resources/libraries/bash/entry/bootstrap_verify_perf.sh"