diff options
| -rwxr-xr-x | bootstrap.sh | 330 | ||||
| -rw-r--r-- | resources/libraries/python/SetupFramework.py | 12 |
2 files changed, 258 insertions, 84 deletions
diff --git a/bootstrap.sh b/bootstrap.sh index f822badde5..2796e5031c 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -22,7 +22,6 @@ sudo apt-get -y update sudo apt-get -y install libpython2.7-dev python-virtualenv VIRL_SERVERS=("10.30.51.28" "10.30.51.29" "10.30.51.30") -VIRL_SERVER="" VIRL_USERNAME=jenkins-in VIRL_PKEY=priv_key @@ -31,6 +30,18 @@ VIRL_SERVER_EXPECTED_STATUS="PRODUCTION" SSH_OPTIONS="-i ${VIRL_PKEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o LogLevel=error" +TEST_GROUPS=("bridge_domain,dhcp,gre,honeycomb,l2_xconnect,lisp,softwire" "cop,ipfix,ipsec,ipv6,rpf,tap,vrf" "fds,iacl,ipv4,policer,vlan,vxlan") +SUITE_PATH="tests.func" + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +LOG_PATH="${SCRIPT_DIR}" + +# Create temp dir for tarballs +mkdir ${SCRIPT_DIR}/../temp +export TMPDIR="${SCRIPT_DIR}/../temp" + +SHARED_MEMORY_PATH="/run/shm" + function ssh_do() { echo echo "### " ssh $@ @@ -70,42 +81,44 @@ EOF chmod 600 ${VIRL_PKEY} # -# Pick a random host from the array of VIRL servers, and attempt -# to reach it and verify it's status. +# Pick a random host from the array of VIRL servers for every test set run +# and attempt to reach it and verify it's status. # -# The server must be reachable, and have a "status" file with -# the content "PRODUCTION", to be selected. +# The server must be reachable and have a "status" file with +# the content "PRODUCTION" to be selected. # -# If the server is not reachable, or does not have the correct -# status, remove it from the array and start again. +# If the server is not reachable or does not have the correct +# status remove it from the array and start again. # # Abort if there are no more servers left in the array. # -while [[ ! "$VIRL_SERVER" ]] -do - num_hosts=${#VIRL_SERVERS[@]} - if [ $num_hosts == 0 ] - then - echo "No more VIRL candidate hosts available, failing." - exit 127 - fi - element=$[ $RANDOM % $num_hosts ] - virl_server_candidate=${VIRL_SERVERS[$element]} - virl_server_status=$(ssh ${SSH_OPTIONS} ${VIRL_USERNAME}@${virl_server_candidate} cat $VIRL_SERVER_STATUS_FILE 2>&1) - echo VIRL HOST $virl_server_candidate status is \"$virl_server_status\" - if [ "$virl_server_status" == "$VIRL_SERVER_EXPECTED_STATUS" ] - then - # Candidate is in good status. Select this server. - VIRL_SERVER="$virl_server_candidate" - else - # Candidate is in bad status. Remove from array. - VIRL_SERVERS=("${VIRL_SERVERS[@]:0:$element}" "${VIRL_SERVERS[@]:$[$element+1]}") - fi +for index in "${!TEST_GROUPS[@]}"; do + VIRL_SERVER[${index}]="" + while [[ ! "${VIRL_SERVER[${index}]}" ]]; do + num_hosts=${#VIRL_SERVERS[@]} + if [ $num_hosts == 0 ] + then + echo "No more VIRL candidate hosts available, failing." + exit 127 + fi + element=$[ $RANDOM % $num_hosts ] + virl_server_candidate=${VIRL_SERVERS[$element]} + virl_server_status=$(ssh ${SSH_OPTIONS} ${VIRL_USERNAME}@${virl_server_candidate} cat $VIRL_SERVER_STATUS_FILE 2>&1) + echo VIRL HOST $virl_server_candidate status is \"$virl_server_status\" + if [ "$virl_server_status" == "$VIRL_SERVER_EXPECTED_STATUS" ] + then + # Candidate is in good status. Select this server. + VIRL_SERVER[${index}]="$virl_server_candidate" + else + # Candidate is in bad status. Remove from array. + VIRL_SERVERS=("${VIRL_SERVERS[@]:0:$element}" "${VIRL_SERVERS[@]:$[$element+1]}") + fi + done done -# Temporarily download VPP packages from nexus.fd.io -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +echo "Selected VIRL servers: ${VIRL_SERVER[@]}" +# Temporarily download VPP packages from nexus.fd.io if [ "${#}" -ne "0" ]; then arr=(${@}) echo ${arr[0]} @@ -136,56 +149,75 @@ done echo "Updated file names: " ${VPP_DEBS_FULL[@]} cat ${VIRL_PKEY} -# Copy the files to VIRL host -scp ${SSH_OPTIONS} *.deb \ - ${VIRL_USERNAME}@${VIRL_SERVER}:${VIRL_DIR_LOC}/ - -result=$? -if [ "${result}" -ne "0" ]; then - echo "Failed to copy vpp deb files to virl host" - echo ${result} - exit ${result} -fi + +# Copy the files to VIRL hosts +DONE="" +for index in "${!VIRL_SERVER[@]}"; do + # Do not copy files in case they have already been copied to the VIRL host + [[ "${DONE[@]}" =~ "${VIRL_SERVER[${index}]}" ]] && copy=0 || copy=1 + + if [ "${copy}" -eq "0" ]; then + echo "VPP deb files have already been copied to the VIRL host ${VIRL_SERVER[${index}]}" + else + scp ${SSH_OPTIONS} *.deb \ + ${VIRL_USERNAME}@${VIRL_SERVER[${index}]}:${VIRL_DIR_LOC}/ + + result=$? + if [ "${result}" -ne "0" ]; then + echo "Failed to copy VPP deb files to VIRL host ${VIRL_SERVER[${index}]}" + echo ${result} + exit ${result} + else + echo "VPP deb files successfully copied to the VIRL host ${VIRL_SERVER[${index}]}" + fi + DONE+=(${VIRL_SERVER[${index}]}) + fi +done # Start a simulation on VIRL server -echo "Starting simulation on VIRL server" function stop_virl_simulation { - ssh ${SSH_OPTIONS} ${VIRL_USERNAME}@${VIRL_SERVER}\ - "stop-testcase ${VIRL_SID}" + for index in "${!VIRL_SERVER[@]}"; do + ssh ${SSH_OPTIONS} ${VIRL_USERNAME}@${VIRL_SERVER[${index}]}\ + "stop-testcase ${VIRL_SID[${index}]}" + done } -VIRL_SID=$(ssh ${SSH_OPTIONS} \ - ${VIRL_USERNAME}@${VIRL_SERVER} \ - "start-testcase -c double-ring-nested ${VPP_DEBS_FULL[@]}") -retval=$? -if [ "$?" -ne "0" ]; then - echo "VIRL simulation start failed" - exit ${retval} -fi - -if [[ ! "${VIRL_SID}" =~ session-[a-zA-Z0-9_]{6} ]]; then - echo "No VIRL session ID reported." - exit 127 -fi - # Upon script exit, cleanup the simulation execution trap stop_virl_simulation EXIT -echo ${VIRL_SID} -ssh_do ${VIRL_USERNAME}@${VIRL_SERVER} cat /scratch/${VIRL_SID}/topology.yaml +for index in "${!VIRL_SERVER[@]}"; do + echo "Starting simulation nr. ${index} on VIRL server ${VIRL_SERVER[${index}]}" + VIRL_SID[${index}]=$(ssh ${SSH_OPTIONS} \ + ${VIRL_USERNAME}@${VIRL_SERVER[${index}]} \ + "start-testcase -c double-ring-nested ${VPP_DEBS_FULL[@]}") + retval=$? + if [ "$?" -ne "0" ]; then + echo "VIRL simulation start failed on ${VIRL_SERVER[${index}]}" + exit ${retval} + fi + if [[ ! "${VIRL_SID[${index}]}" =~ session-[a-zA-Z0-9_]{6} ]]; then + echo "No VIRL session ID reported." + exit 127 + fi + echo "VIRL simulation started on ${VIRL_SERVER[${index}]}" + + ssh_do ${VIRL_USERNAME}@${VIRL_SERVER[${index}]}\ + cat /scratch/${VIRL_SID[${index}]}/topology.yaml -# Download the topology file from virl session -scp ${SSH_OPTIONS} \ - ${VIRL_USERNAME}@${VIRL_SERVER}:/scratch/${VIRL_SID}/topology.yaml \ - topologies/enabled/topology.yaml + # Download the topology file from VIRL session and rename it + scp ${SSH_OPTIONS} \ + ${VIRL_USERNAME}@${VIRL_SERVER[${index}]}:/scratch/${VIRL_SID[${index}]}/topology.yaml \ + topologies/enabled/topology${index}.yaml -retval=$? -if [ "$?" -ne "0" ]; then - echo "Failed to copy topology file from VIRL simulation" - exit ${retval} -fi + retval=$? + if [ "$?" -ne "0" ]; then + echo "Failed to copy topology file from VIRL simulation nr. ${index} on VIRL server ${VIRL_SERVER[${index}]}" + exit ${retval} + fi +done +echo ${VIRL_SID[@]} virtualenv --system-site-packages env . env/bin/activate @@ -193,22 +225,154 @@ virtualenv --system-site-packages env echo pip install pip install -r ${SCRIPT_DIR}/requirements.txt -pykwalify -s ${SCRIPT_DIR}/resources/topology_schemas/3_node_topology.sch.yaml \ - -s ${SCRIPT_DIR}/resources/topology_schemas/topology.sch.yaml \ - -d ${SCRIPT_DIR}/topologies/enabled/topology.yaml \ - -vvv +for index in "${!VIRL_SERVER[@]}"; do + pykwalify -s ${SCRIPT_DIR}/resources/topology_schemas/3_node_topology.sch.yaml \ + -s ${SCRIPT_DIR}/resources/topology_schemas/topology.sch.yaml \ + -d ${SCRIPT_DIR}/topologies/enabled/topology${index}.yaml \ + -vvv + if [ "$?" -ne "0" ]; then + echo "Topology${index} schema validation failed." + echo "However, the tests will start." + fi +done + +function run_test_set() { + set +x + OLDIFS=$IFS + IFS="," + nr=$(echo $1) + rm -f ${LOG_PATH}/test_run${nr}.log + exec &> >(while read line; do echo "$(date +'%H:%M:%S') $line" \ + >> ${LOG_PATH}/test_run${nr}.log; done;) + suite_str="" + for suite in ${TEST_GROUPS[${nr}]}; do + suite_str="${suite_str} --suite ${SUITE_PATH}.${suite}" + done + IFS=$OLDIFS -if [ "$?" -ne "0" ]; then - echo "Topology schema validation failed." - echo "However, the tests will start." + echo "PYTHONPATH=`pwd` pybot -L TRACE -W 136\ + -v TOPOLOGY_PATH:${SCRIPT_DIR}/topologies/enabled/topology${nr}.yaml \ + ${suite_str} \ + --include vm_envAND3_node_single_link_topo \ + --include vm_envAND3_node_double_link_topo \ + --exclude PERFTEST \ + --exclude SKIP_PATCH \ + --noncritical EXPECTED_FAILING \ + --output ${LOG_PATH}/log_test_set_run${nr} \ + tests/" + + PYTHONPATH=`pwd` pybot -L TRACE -W 136\ + -v TOPOLOGY_PATH:${SCRIPT_DIR}/topologies/enabled/topology${nr}.yaml \ + ${suite_str} \ + --include vm_envAND3_node_single_link_topo \ + --include vm_envAND3_node_double_link_topo \ + --exclude PERFTEST \ + --exclude SKIP_PATCH \ + --noncritical EXPECTED_FAILING \ + --output ${LOG_PATH}/log_test_set_run${nr} \ + tests/ + + local_run_rc=$? + echo ${local_run_rc} > ${SHARED_MEMORY_PATH}/rc_test_run${nr} + set -x +} + +set +x +# Send to background an instance of the run_test_set() function for each number, +# record the pid. +for index in "${!VIRL_SERVER[@]}"; do + run_test_set ${index} & + pid=$! + echo "Sent to background: Test_set${index} (pid=$pid)" + pids[$pid]=$index +done + +echo +echo -n "Waiting..." + +# Watch the stable of background processes. +# If a pid goes away, remove it from the array. +while [ -n "${pids[*]}" ]; do + for i in $(seq 0 9); do + sleep 1 + echo -n "." + done + for pid in "${!pids[@]}"; do + if ! ps "$pid" >/dev/null; then + echo -e "\n" + echo "Test_set${pids[$pid]} with PID $pid finished." + unset pids[$pid] + fi + done + if [ -z "${!pids[*]}" ]; then + break + fi + echo -n -e "\nStill waiting for test set(s): ${pids[*]} ..." +done + +echo +echo "All test set runs finished." +echo + +set -x + +RC=0 +for index in "${!VIRL_SERVER[@]}"; do + echo "Test_set${index} log:" + cat ${LOG_PATH}/test_run${index}.log + RC_PARTIAL_RUN=$(cat ${SHARED_MEMORY_PATH}/rc_test_run${index}) + RC=$((RC+RC_PARTIAL_RUN)) + rm -f ${SHARED_MEMORY_PATH}/rc_test_run${index} + rm -f ${LOG_PATH}/test_run${index}.log + echo +done + +# Log the final result +if [ "${RC}" -eq "0" ]; then + set +x + echo + echo "========================================================================================================================================" + echo "Final result of all test loops: | PASS |" + echo "All critical tests have passed." + echo "========================================================================================================================================" + echo + set -x +else + if [ "${RC}" -eq "1" ]; then + HLP_STR="test has" + else + HLP_STR="tests have" + fi + set +x + echo + echo "========================================================================================================================================" + echo "Final result of all test loops: | FAIL |" + echo "${RC} critical ${HLP_STR} failed." + echo "========================================================================================================================================" + echo + set -x +fi + +echo Post-processing test data... + +partial_logs="" +for index in "${!VIRL_SERVER[@]}"; do + partial_logs="${partial_logs} ${LOG_PATH}/log_test_set_run${index}.xml" +done + +# Rebot output post-processing +rebot --noncritical EXPECTED_FAILING \ + --output output.xml ${partial_logs} + +# Remove unnecessary log files +rm -f ${partial_logs} + +echo Post-processing finished. + +if [ ${RC} -eq 0 ]; then + RETURN_STATUS=0 +else + RETURN_STATUS=1 fi -PYTHONPATH=`pwd` pybot -L TRACE -W 136\ - -v TOPOLOGY_PATH:${SCRIPT_DIR}/topologies/enabled/topology.yaml \ - --suite "tests.func" \ - --include vm_envAND3_node_single_link_topo \ - --include vm_envAND3_node_double_link_topo \ - --exclude PERFTEST \ - --exclude SKIP_PATCH \ - --noncritical EXPECTED_FAILING \ - tests/ +exit ${RETURN_STATUS} diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py index 570a2baf4b..d035317006 100644 --- a/resources/libraries/python/SetupFramework.py +++ b/resources/libraries/python/SetupFramework.py @@ -21,6 +21,7 @@ from subprocess import Popen, PIPE, call from multiprocessing import Pool from tempfile import NamedTemporaryFile from os.path import basename +from os import environ from robot.api import logger from robot.libraries.BuiltIn import BuiltIn @@ -35,7 +36,16 @@ __all__ = ["SetupFramework"] def pack_framework_dir(): """Pack the testing WS into temp file, return its name.""" - tmpfile = NamedTemporaryFile(suffix=".tgz", prefix="openvpp-testing-") + try: + directory = environ["TMPDIR"] + except Keyerror: + directory = None + + if directory is not None: + tmpfile = NamedTemporaryFile(suffix=".tgz", prefix="openvpp-testing-",\ + dir="{0}".format(directory)) + else: + tmpfile = NamedTemporaryFile(suffix=".tgz", prefix="openvpp-testing-") file_name = tmpfile.name tmpfile.close() |