From 26c10238dc24b81ff34cf2a85042908be338373a Mon Sep 17 00:00:00 2001 From: Jan Date: Sun, 25 Sep 2016 22:29:18 +0200 Subject: Use tmp directory for log files - store log files in tmp directory and exclude all files in this directory from archiving when creating tarballs Change-Id: I1e2dde76dafb533678774733c15c17e7e6ec022d Signed-off-by: Jan Gelety --- bootstrap.sh | 334 ++++++++++++++++++++------- resources/libraries/python/SetupFramework.py | 27 ++- 2 files changed, 271 insertions(+), 90 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index f822badde5..2bd3a0f072 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -22,7 +22,6 @@ sudo apt-get -y update sudo apt-get -y install libpython2.7-dev python-virtualenv VIRL_SERVERS=("10.30.51.28" "10.30.51.29" "10.30.51.30") -VIRL_SERVER="" VIRL_USERNAME=jenkins-in VIRL_PKEY=priv_key @@ -31,6 +30,22 @@ VIRL_SERVER_EXPECTED_STATUS="PRODUCTION" SSH_OPTIONS="-i ${VIRL_PKEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o LogLevel=error" +TEST_GROUPS=("bridge_domain,dhcp,gre,honeycomb,l2_xconnect,lisp,softwire" "cop,ipfix,ipsec,ipv6,rpf,tap,vrf" "fds,iacl,ipv4,policer,vlan,vxlan") +SUITE_PATH="tests.func" + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Create tmp dir +mkdir ${SCRIPT_DIR}/tmp + +# Use tmp dir to store log files +LOG_PATH="${SCRIPT_DIR}/tmp" + +# Use tmp dir for tarballs +export TMPDIR="${SCRIPT_DIR}/tmp" + +SHARED_MEMORY_PATH="/run/shm" + function ssh_do() { echo echo "### " ssh $@ @@ -70,42 +85,44 @@ EOF chmod 600 ${VIRL_PKEY} # -# Pick a random host from the array of VIRL servers, and attempt -# to reach it and verify it's status. +# Pick a random host from the array of VIRL servers for every test set run +# and attempt to reach it and verify it's status. # -# The server must be reachable, and have a "status" file with -# the content "PRODUCTION", to be selected. +# The server must be reachable and have a "status" file with +# the content "PRODUCTION" to be selected. # -# If the server is not reachable, or does not have the correct -# status, remove it from the array and start again. +# If the server is not reachable or does not have the correct +# status remove it from the array and start again. # # Abort if there are no more servers left in the array. # -while [[ ! "$VIRL_SERVER" ]] -do - num_hosts=${#VIRL_SERVERS[@]} - if [ $num_hosts == 0 ] - then - echo "No more VIRL candidate hosts available, failing." - exit 127 - fi - element=$[ $RANDOM % $num_hosts ] - virl_server_candidate=${VIRL_SERVERS[$element]} - virl_server_status=$(ssh ${SSH_OPTIONS} ${VIRL_USERNAME}@${virl_server_candidate} cat $VIRL_SERVER_STATUS_FILE 2>&1) - echo VIRL HOST $virl_server_candidate status is \"$virl_server_status\" - if [ "$virl_server_status" == "$VIRL_SERVER_EXPECTED_STATUS" ] - then - # Candidate is in good status. Select this server. - VIRL_SERVER="$virl_server_candidate" - else - # Candidate is in bad status. Remove from array. - VIRL_SERVERS=("${VIRL_SERVERS[@]:0:$element}" "${VIRL_SERVERS[@]:$[$element+1]}") - fi +for index in "${!TEST_GROUPS[@]}"; do + VIRL_SERVER[${index}]="" + while [[ ! "${VIRL_SERVER[${index}]}" ]]; do + num_hosts=${#VIRL_SERVERS[@]} + if [ $num_hosts == 0 ] + then + echo "No more VIRL candidate hosts available, failing." + exit 127 + fi + element=$[ $RANDOM % $num_hosts ] + virl_server_candidate=${VIRL_SERVERS[$element]} + virl_server_status=$(ssh ${SSH_OPTIONS} ${VIRL_USERNAME}@${virl_server_candidate} cat $VIRL_SERVER_STATUS_FILE 2>&1) + echo VIRL HOST $virl_server_candidate status is \"$virl_server_status\" + if [ "$virl_server_status" == "$VIRL_SERVER_EXPECTED_STATUS" ] + then + # Candidate is in good status. Select this server. + VIRL_SERVER[${index}]="$virl_server_candidate" + else + # Candidate is in bad status. Remove from array. + VIRL_SERVERS=("${VIRL_SERVERS[@]:0:$element}" "${VIRL_SERVERS[@]:$[$element+1]}") + fi + done done -# Temporarily download VPP packages from nexus.fd.io -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +echo "Selected VIRL servers: ${VIRL_SERVER[@]}" +# Temporarily download VPP packages from nexus.fd.io if [ "${#}" -ne "0" ]; then arr=(${@}) echo ${arr[0]} @@ -136,56 +153,75 @@ done echo "Updated file names: " ${VPP_DEBS_FULL[@]} cat ${VIRL_PKEY} -# Copy the files to VIRL host -scp ${SSH_OPTIONS} *.deb \ - ${VIRL_USERNAME}@${VIRL_SERVER}:${VIRL_DIR_LOC}/ - -result=$? -if [ "${result}" -ne "0" ]; then - echo "Failed to copy vpp deb files to virl host" - echo ${result} - exit ${result} -fi + +# Copy the files to VIRL hosts +DONE="" +for index in "${!VIRL_SERVER[@]}"; do + # Do not copy files in case they have already been copied to the VIRL host + [[ "${DONE[@]}" =~ "${VIRL_SERVER[${index}]}" ]] && copy=0 || copy=1 + + if [ "${copy}" -eq "0" ]; then + echo "VPP deb files have already been copied to the VIRL host ${VIRL_SERVER[${index}]}" + else + scp ${SSH_OPTIONS} *.deb \ + ${VIRL_USERNAME}@${VIRL_SERVER[${index}]}:${VIRL_DIR_LOC}/ + + result=$? + if [ "${result}" -ne "0" ]; then + echo "Failed to copy VPP deb files to VIRL host ${VIRL_SERVER[${index}]}" + echo ${result} + exit ${result} + else + echo "VPP deb files successfully copied to the VIRL host ${VIRL_SERVER[${index}]}" + fi + DONE+=(${VIRL_SERVER[${index}]}) + fi +done # Start a simulation on VIRL server -echo "Starting simulation on VIRL server" function stop_virl_simulation { - ssh ${SSH_OPTIONS} ${VIRL_USERNAME}@${VIRL_SERVER}\ - "stop-testcase ${VIRL_SID}" + for index in "${!VIRL_SERVER[@]}"; do + ssh ${SSH_OPTIONS} ${VIRL_USERNAME}@${VIRL_SERVER[${index}]}\ + "stop-testcase ${VIRL_SID[${index}]}" + done } -VIRL_SID=$(ssh ${SSH_OPTIONS} \ - ${VIRL_USERNAME}@${VIRL_SERVER} \ - "start-testcase -c double-ring-nested ${VPP_DEBS_FULL[@]}") -retval=$? -if [ "$?" -ne "0" ]; then - echo "VIRL simulation start failed" - exit ${retval} -fi - -if [[ ! "${VIRL_SID}" =~ session-[a-zA-Z0-9_]{6} ]]; then - echo "No VIRL session ID reported." - exit 127 -fi - # Upon script exit, cleanup the simulation execution trap stop_virl_simulation EXIT -echo ${VIRL_SID} -ssh_do ${VIRL_USERNAME}@${VIRL_SERVER} cat /scratch/${VIRL_SID}/topology.yaml +for index in "${!VIRL_SERVER[@]}"; do + echo "Starting simulation nr. ${index} on VIRL server ${VIRL_SERVER[${index}]}" + VIRL_SID[${index}]=$(ssh ${SSH_OPTIONS} \ + ${VIRL_USERNAME}@${VIRL_SERVER[${index}]} \ + "start-testcase -c double-ring-nested ${VPP_DEBS_FULL[@]}") + retval=$? + if [ "$?" -ne "0" ]; then + echo "VIRL simulation start failed on ${VIRL_SERVER[${index}]}" + exit ${retval} + fi + if [[ ! "${VIRL_SID[${index}]}" =~ session-[a-zA-Z0-9_]{6} ]]; then + echo "No VIRL session ID reported." + exit 127 + fi + echo "VIRL simulation started on ${VIRL_SERVER[${index}]}" -# Download the topology file from virl session -scp ${SSH_OPTIONS} \ - ${VIRL_USERNAME}@${VIRL_SERVER}:/scratch/${VIRL_SID}/topology.yaml \ - topologies/enabled/topology.yaml + ssh_do ${VIRL_USERNAME}@${VIRL_SERVER[${index}]}\ + cat /scratch/${VIRL_SID[${index}]}/topology.yaml -retval=$? -if [ "$?" -ne "0" ]; then - echo "Failed to copy topology file from VIRL simulation" - exit ${retval} -fi + # Download the topology file from VIRL session and rename it + scp ${SSH_OPTIONS} \ + ${VIRL_USERNAME}@${VIRL_SERVER[${index}]}:/scratch/${VIRL_SID[${index}]}/topology.yaml \ + topologies/enabled/topology${index}.yaml + + retval=$? + if [ "$?" -ne "0" ]; then + echo "Failed to copy topology file from VIRL simulation nr. ${index} on VIRL server ${VIRL_SERVER[${index}]}" + exit ${retval} + fi +done +echo ${VIRL_SID[@]} virtualenv --system-site-packages env . env/bin/activate @@ -193,22 +229,154 @@ virtualenv --system-site-packages env echo pip install pip install -r ${SCRIPT_DIR}/requirements.txt -pykwalify -s ${SCRIPT_DIR}/resources/topology_schemas/3_node_topology.sch.yaml \ - -s ${SCRIPT_DIR}/resources/topology_schemas/topology.sch.yaml \ - -d ${SCRIPT_DIR}/topologies/enabled/topology.yaml \ - -vvv +for index in "${!VIRL_SERVER[@]}"; do + pykwalify -s ${SCRIPT_DIR}/resources/topology_schemas/3_node_topology.sch.yaml \ + -s ${SCRIPT_DIR}/resources/topology_schemas/topology.sch.yaml \ + -d ${SCRIPT_DIR}/topologies/enabled/topology${index}.yaml \ + -vvv + if [ "$?" -ne "0" ]; then + echo "Topology${index} schema validation failed." + echo "However, the tests will start." + fi +done -if [ "$?" -ne "0" ]; then - echo "Topology schema validation failed." - echo "However, the tests will start." +function run_test_set() { + set +x + OLDIFS=$IFS + IFS="," + nr=$(echo $1) + rm -f ${LOG_PATH}/test_run${nr}.log + exec &> >(while read line; do echo "$(date +'%H:%M:%S') $line" \ + >> ${LOG_PATH}/test_run${nr}.log; done;) + suite_str="" + for suite in ${TEST_GROUPS[${nr}]}; do + suite_str="${suite_str} --suite ${SUITE_PATH}.${suite}" + done + IFS=$OLDIFS + + echo "PYTHONPATH=`pwd` pybot -L TRACE -W 136\ + -v TOPOLOGY_PATH:${SCRIPT_DIR}/topologies/enabled/topology${nr}.yaml \ + ${suite_str} \ + --include vm_envAND3_node_single_link_topo \ + --include vm_envAND3_node_double_link_topo \ + --exclude PERFTEST \ + --exclude SKIP_PATCH \ + --noncritical EXPECTED_FAILING \ + --output ${LOG_PATH}/log_test_set_run${nr} \ + tests/" + + PYTHONPATH=`pwd` pybot -L TRACE -W 136\ + -v TOPOLOGY_PATH:${SCRIPT_DIR}/topologies/enabled/topology${nr}.yaml \ + ${suite_str} \ + --include vm_envAND3_node_single_link_topo \ + --include vm_envAND3_node_double_link_topo \ + --exclude PERFTEST \ + --exclude SKIP_PATCH \ + --noncritical EXPECTED_FAILING \ + --output ${LOG_PATH}/log_test_set_run${nr} \ + tests/ + + local_run_rc=$? + echo ${local_run_rc} > ${SHARED_MEMORY_PATH}/rc_test_run${nr} + set -x +} + +set +x +# Send to background an instance of the run_test_set() function for each number, +# record the pid. +for index in "${!VIRL_SERVER[@]}"; do + run_test_set ${index} & + pid=$! + echo "Sent to background: Test_set${index} (pid=$pid)" + pids[$pid]=$index +done + +echo +echo -n "Waiting..." + +# Watch the stable of background processes. +# If a pid goes away, remove it from the array. +while [ -n "${pids[*]}" ]; do + for i in $(seq 0 9); do + sleep 1 + echo -n "." + done + for pid in "${!pids[@]}"; do + if ! ps "$pid" >/dev/null; then + echo -e "\n" + echo "Test_set${pids[$pid]} with PID $pid finished." + unset pids[$pid] + fi + done + if [ -z "${!pids[*]}" ]; then + break + fi + echo -n -e "\nStill waiting for test set(s): ${pids[*]} ..." +done + +echo +echo "All test set runs finished." +echo + +set -x + +RC=0 +for index in "${!VIRL_SERVER[@]}"; do + echo "Test_set${index} log:" + cat ${LOG_PATH}/test_run${index}.log + RC_PARTIAL_RUN=$(cat ${SHARED_MEMORY_PATH}/rc_test_run${index}) + RC=$((RC+RC_PARTIAL_RUN)) + rm -f ${SHARED_MEMORY_PATH}/rc_test_run${index} + rm -f ${LOG_PATH}/test_run${index}.log + echo +done + +# Log the final result +if [ "${RC}" -eq "0" ]; then + set +x + echo + echo "========================================================================================================================================" + echo "Final result of all test loops: | PASS |" + echo "All critical tests have passed." + echo "========================================================================================================================================" + echo + set -x +else + if [ "${RC}" -eq "1" ]; then + HLP_STR="test has" + else + HLP_STR="tests have" + fi + set +x + echo + echo "========================================================================================================================================" + echo "Final result of all test loops: | FAIL |" + echo "${RC} critical ${HLP_STR} failed." + echo "========================================================================================================================================" + echo + set -x +fi + +echo Post-processing test data... + +partial_logs="" +for index in "${!VIRL_SERVER[@]}"; do + partial_logs="${partial_logs} ${LOG_PATH}/log_test_set_run${index}.xml" +done + +# Rebot output post-processing +rebot --noncritical EXPECTED_FAILING \ + --output output.xml ${partial_logs} + +# Remove unnecessary log files +rm -f ${partial_logs} + +echo Post-processing finished. + +if [ ${RC} -eq 0 ]; then + RETURN_STATUS=0 +else + RETURN_STATUS=1 fi -PYTHONPATH=`pwd` pybot -L TRACE -W 136\ - -v TOPOLOGY_PATH:${SCRIPT_DIR}/topologies/enabled/topology.yaml \ - --suite "tests.func" \ - --include vm_envAND3_node_single_link_topo \ - --include vm_envAND3_node_double_link_topo \ - --exclude PERFTEST \ - --exclude SKIP_PATCH \ - --noncritical EXPECTED_FAILING \ - tests/ +exit ${RETURN_STATUS} diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py index 570a2baf4b..1a1e991b3b 100644 --- a/resources/libraries/python/SetupFramework.py +++ b/resources/libraries/python/SetupFramework.py @@ -21,6 +21,7 @@ from subprocess import Popen, PIPE, call from multiprocessing import Pool from tempfile import NamedTemporaryFile from os.path import basename +from os import environ from robot.api import logger from robot.libraries.BuiltIn import BuiltIn @@ -35,13 +36,22 @@ __all__ = ["SetupFramework"] def pack_framework_dir(): """Pack the testing WS into temp file, return its name.""" - tmpfile = NamedTemporaryFile(suffix=".tgz", prefix="openvpp-testing-") + try: + directory = environ["TMPDIR"] + except KeyError: + directory = None + + if directory is not None: + tmpfile = NamedTemporaryFile(suffix=".tgz", prefix="openvpp-testing-", + dir="{0}".format(directory)) + else: + tmpfile = NamedTemporaryFile(suffix=".tgz", prefix="openvpp-testing-") file_name = tmpfile.name tmpfile.close() proc = Popen( - split("tar --exclude-vcs -zcf {0} .".format(file_name)), - stdout=PIPE, stderr=PIPE) + split("tar --exclude-vcs --exclude=./tmp --exclude=*.deb -zcf {0} .". + format(file_name)), stdout=PIPE, stderr=PIPE) (stdout, stderr) = proc.communicate() logger.debug(stdout) @@ -73,7 +83,7 @@ def copy_tarball_to_node(tarball, node): def extract_tarball_at_node(tarball, node): """Extract tarball at given node. - Extracts tarball using tar on given node to specific CSIT loocation. + Extracts tarball using tar on given node to specific CSIT location. :param tarball: Path to tarball to upload. :param node: Dictionary created from topology. @@ -102,7 +112,8 @@ def create_env_directory_at_node(node): ssh = SSH() ssh.connect(node) (ret_code, stdout, stderr) = ssh.exec_command( - 'cd {0} && rm -rf env && virtualenv --system-site-packages env && . env/bin/activate && ' + 'cd {0} && rm -rf env && virtualenv --system-site-packages env && ' + '. env/bin/activate && ' 'pip install -r requirements.txt' .format(con.REMOTE_FW_DIR), timeout=100) if 0 != ret_code: @@ -111,7 +122,8 @@ def create_env_directory_at_node(node): else: logger.console('Virtualenv created on {0}'.format(node['host'])) -#pylint: disable=broad-except + +# pylint: disable=broad-except def setup_node(args): """Run all set-up methods for a node. @@ -137,6 +149,7 @@ def setup_node(args): logger.console('Setup of node {0} done'.format(node['host'])) return True + def delete_local_tarball(tarball): """Delete local tarball to prevent disk pollution. @@ -147,7 +160,7 @@ def delete_local_tarball(tarball): call(split('sh -c "rm {0} > /dev/null 2>&1"'.format(tarball))) -class SetupFramework(object): # pylint: disable=too-few-public-methods +class SetupFramework(object): # pylint: disable=too-few-public-methods """Setup suite run on topology nodes. Many VAT/CLI based tests need the scripts at remote hosts before executing -- cgit 1.2.3-korg