diff options
| -rw-r--r-- | bootstrap-vpp-honeycomb.sh | 173 | ||||
| -rw-r--r-- | resources/libraries/python/honeycomb/HoneycombSetup.py | 54 | ||||
| -rw-r--r-- | resources/tools/testbed-setup/playbooks/03-virl-post-install.yaml | 4 | ||||
| -rwxr-xr-x | resources/tools/virl/bin/start-honeycomb-testcase | 375 | ||||
| -rw-r--r-- | virl_params.sh | 22 |
5 files changed, 628 insertions, 0 deletions
diff --git a/bootstrap-vpp-honeycomb.sh b/bootstrap-vpp-honeycomb.sh new file mode 100644 index 0000000000..2e89d9d747 --- /dev/null +++ b/bootstrap-vpp-honeycomb.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# Copyright (c) 2016 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +cat /etc/hostname +cat /etc/hosts + +export DEBIAN_FRONTEND=noninteractive +sudo apt-get -y update +sudo apt-get -y install libpython2.7-dev python-virtualenv + +# Source the VIRL server parameters: +source virl_params.sh + +SSH_OPTIONS="-i ${VIRL_PKEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o LogLevel=error" + +function ssh_do() { + echo + echo "### " ssh $@ + ssh ${SSH_OPTIONS} $@ +} + +rm -f ${VIRL_PKEY} +cat > ${VIRL_PKEY} <<EOF +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEA+IHXq87GcqMR1C47rzx6Cbip5Ghq8pKrbqKrP5Nf41HcYrT6 +GOXl9nFWKsMOzIlIn+8y7Il27eZh7csQGApbg8QLiHMtcYEmWNzKZpkqg4nuAPxX +VXwlKgnKX902SrET9Gp9TDayiHtCRWVfrlPPPSA0UEXW6BjLN/uHJ+W/Xzrrab+9 +asBVa05vT2W6n0KJ66zfCaeDM912mQ6SttscAwFoWDmdHlegiVqrlIG2ABxOvxxz +L3dM3iSmlmQlzv9bThjo+nI4KFYh6m5wrZmAo5r/4q9CIJc21HVnTqkGOWJIZz6J +73lePJVSq5gYqaoGw3swFEA/MDkOx7baWKSoLQIDAQABAoIBAQCNBeolNp+JWJ76 +gQ4fwLsknyXSV6sxYyhkDW4PEwwcTU06uqce0AAzXVffxne0fMe48x47+zqBgPbb +4huM+Pu8B9nfojUMr5TaYtl9Zbgpk3F8H7dT7LKOa6XrxvZTZrADSRc30+Z26zPN +e9zTaf42Gvt0/l0Zs1BHwbaOXqO+XuwJ3/F9Sf3PQYWXD3EOWjpHDP/X/1vAs6lV +SLkm6J/9KKE1m6I6LTYjIXuYt4SXybW6N2TSy54hhQtYcDUnIU2hR/PHVWKrGA0J +kELgrtTNTdbML27O5gFWU4PLUEYTZ9fN11D6qUZKxLcPOiPPHXkiILMRCCnG5DYI +ksBAU/YlAoGBAPxZO9VO18TYc8THV1nLKcvT2+1oSs1UcA2wNQMU55t910ZYinRa +MRwUhMOf8Mv5wOeiZaRICQB1PnVWtDVmGECgPpK6jUxqAwn8rgJcnoafLGL5YKMY +RVafTe6N5LXgCaOcJrk21wxs6v7ninEbUxxc575urOvZMBkymDw91dwbAoGBAPwa +YRhKhrzFKZzdK0RadVjnxKvolUllpoqqg3XuvmeAJHAOAnaOgVWq68NAcp5FZJv0 +2D2Up7TX8pjf9MofP1SJbcraKBpK4NzfNkA0dSdEi+FhVofAJ9umB2o5LW1n7sab +UIrjsdzSJK/9Zb9yTTHPyibYzNEgaJV1HsbxfEFXAoGAYO2RmvRm0phll18OQVJV +IpKk9kLKAKZ/R/K32hAsikBC8SVPQTPniyaifFWx81diblalff2hX4ipTf7Yx24I +wMIMZuW7Im/R7QMef4+94G3Bad7p7JuE/qnAEHJ2OBnu+eYfxaK35XDsrq6XMazS +NqHE7hOq3giVfgg+C12hCKMCgYEAtu9dbYcG5owbehxzfRI2/OCRsjz/t1bv1seM +xVMND4XI6xb/apBWAZgZpIFrqrWoIBM3ptfsKipZe91ngBPUnL9s0Dolx452RVAj +yctHB8uRxWYgqDkjsxtzXf1HnZBBkBS8CUzYj+hdfuddoeKLaY3invXLCiV+PpXS +U4KAK9kCgYEAtSv0m5+Fg74BbAiFB6kCh11FYkW94YI6B/E2D/uVTD5dJhyEUFgZ +cWsudXjMki8734WSpMBqBp/J8wG3C9ZS6IpQD+U7UXA+roB7Qr+j4TqtWfM+87Rh +maOpG56uAyR0w5Z9BhwzA3VakibVk9KwDgZ29WtKFzuATLFnOtCS46E= +-----END RSA PRIVATE KEY----- +EOF +chmod 600 ${VIRL_PKEY} + +# +# Pick a random host from the array of VIRL servers, and attempt +# to reach it and verify it's status. +# +# The server must be reachable, and have a "status" file with +# the content "PRODUCTION", to be selected. +# +# If the server is not reachable, or does not have the correct +# status, remove it from the array and start again. +# +# Abort if there are no more servers left in the array. +# +while [[ ! "$VIRL_SERVER" ]] +do + num_hosts=${#VIRL_SERVERS[@]} + if [ $num_hosts == 0 ] + then + echo "No more VIRL candidate hosts available, failing." + exit 127 + fi + element=$[ $RANDOM % $num_hosts ] + virl_server_candidate=${VIRL_SERVERS[$element]} + virl_server_status=$(ssh ${SSH_OPTIONS} ${VIRL_USERNAME}@${virl_server_candidate} cat $VIRL_SERVER_STATUS_FILE 2>&1) + echo VIRL HOST $virl_server_candidate status is \"$virl_server_status\" + if [ "$virl_server_status" == "$VIRL_SERVER_EXPECTED_STATUS" ] + then + # Candidate is in good status. Select this server. + VIRL_SERVER="$virl_server_candidate" + else + # Candidate is in bad status. Remove from array. + VIRL_SERVERS=("${VIRL_SERVERS[@]:0:$element}" "${VIRL_SERVERS[@]:$[$element+1]}") + fi +done + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cat ${VIRL_PKEY} + +# Copy start-honeycomb-testcase to VIRL +START_FILE="resources/tools/virl/bin/start-honeycomb-testcase" +DST_DIR_1="/home/jenkins-in/testcase-infra/bin/" +DST_DIR_2="/home/jenkins-in/bin/" + +scp ${SSH_OPTIONS} ${START_FILE} ${VIRL_USERNAME}@${VIRL_SERVER}:${DST_DIR_1} +scp ${SSH_OPTIONS} ${START_FILE} ${VIRL_USERNAME}@${VIRL_SERVER}:${DST_DIR_2} + +# Start a simulation on VIRL server +echo "Starting simulation on VIRL server" + +function stop_virl_simulation { + ssh ${SSH_OPTIONS} ${VIRL_USERNAME}@${VIRL_SERVER}\ + "stop-testcase ${VIRL_SID}" +} + +VIRL_SID=$(ssh ${SSH_OPTIONS} \ + ${VIRL_USERNAME}@${VIRL_SERVER} \ + "start-honeycomb-testcase double-ring-nested") +retval=$? +if [ "$?" -ne "0" ]; then + echo "VIRL simulation start failed" + exit ${retval} +fi + +if [[ ! "${VIRL_SID}" =~ session-[a-zA-Z0-9_]{6} ]]; then + echo "No VIRL session ID reported." + exit 127 +fi + +# Upon script exit, cleanup the simulation execution +trap stop_virl_simulation EXIT +echo ${VIRL_SID} + +ssh_do ${VIRL_USERNAME}@${VIRL_SERVER} cat /scratch/${VIRL_SID}/topology.yaml + +# Download the topology file from virl session +scp ${SSH_OPTIONS} \ + ${VIRL_USERNAME}@${VIRL_SERVER}:/scratch/${VIRL_SID}/topology.yaml \ + topologies/enabled/topology.yaml + +retval=$? +if [ "$?" -ne "0" ]; then + echo "Failed to copy topology file from VIRL simulation" + exit ${retval} +fi + +virtualenv --system-site-packages env +. env/bin/activate + +echo pip install +pip install -r ${SCRIPT_DIR}/requirements.txt + +pykwalify -s ${SCRIPT_DIR}/resources/topology_schemas/3_node_topology.sch.yaml \ + -s ${SCRIPT_DIR}/resources/topology_schemas/topology.sch.yaml \ + -d ${SCRIPT_DIR}/topologies/enabled/topology.yaml \ + -vvv + +if [ "$?" -ne "0" ]; then + echo "Topology schema validation failed." + echo "However, the tests will start." +fi + +PYTHONPATH=`pwd` pybot -L TRACE -W 136\ + -v TOPOLOGY_PATH:${SCRIPT_DIR}/topologies/enabled/topology.yaml \ + --suite "tests.func" \ + --include honeycomb_sanity \ + --noncritical EXPECTED_FAILING \ + tests/ diff --git a/resources/libraries/python/honeycomb/HoneycombSetup.py b/resources/libraries/python/honeycomb/HoneycombSetup.py index 1c232f08a5..04af9a5a8b 100644 --- a/resources/libraries/python/honeycomb/HoneycombSetup.py +++ b/resources/libraries/python/honeycomb/HoneycombSetup.py @@ -53,6 +53,9 @@ class HoneycombSetup(object): :type nodes: list :raises HoneycombError: If Honeycomb fails to start. """ + + HoneycombSetup.print_environment(nodes) + logger.console("\nStarting Honeycomb service ...") cmd = "{0}/bin/start".format(Const.REMOTE_HC_DIR) @@ -123,6 +126,7 @@ class HoneycombSetup(object): for node in nodes: if node['type'] == NodeType.DUT: + HoneycombSetup.print_ports(node) status_code, _ = HTTPRequest.get(node, path, timeout=10, enable_logging=False) if status_code == HTTPCodes.OK: @@ -185,3 +189,53 @@ class HoneycombSetup(object): logger.info("Honeycomb on node {0} has stopped". format(node['host'])) return True + + @staticmethod + def print_environment(nodes): + """Print information about the nodes to log. The information is defined + by commands in cmds tuple at the beginning of this method. + + :param nodes: List of DUT nodes to get information about. + :type nodes: list + """ + + # TODO: When everything is set and running in VIRL env, transform this + # method to a keyword checking the environment. + + cmds = ("uname -a", + "df -lh", + "echo $JAVA_HOME", + "echo $PATH", + "which java", + "java -version", + "dpkg --list | grep openjdk", + "ls -la /opt/honeycomb", + "ls -la /opt/honeycomb/v3po-karaf-1.0.0-SNAPSHOT") + + for node in nodes: + if node['type'] == NodeType.DUT: + logger.info("Checking node {} ...".format(node['host'])) + for cmd in cmds: + logger.info("Command: {}".format(cmd)) + ssh = SSH() + ssh.connect(node) + ssh.exec_command_sudo(cmd) + + @staticmethod + def print_ports(node): + """Uses "sudo netstat -anp | grep java" to print port where a java + application listens. + + :param node: Honeycomb node where we want to print the ports. + :type node: dict + """ + + cmds = ("netstat -anp | grep java", + "ps -ef | grep karaf") + + logger.info("Checking node {} ...".format(node['host'])) + for cmd in cmds: + logger.info("Command: {}".format(cmd)) + ssh = SSH() + ssh.connect(node) + ssh.exec_command_sudo(cmd) diff --git a/resources/tools/testbed-setup/playbooks/03-virl-post-install.yaml b/resources/tools/testbed-setup/playbooks/03-virl-post-install.yaml index 59bcfe9d65..b0e857ca57 100644 --- a/resources/tools/testbed-setup/playbooks/03-virl-post-install.yaml +++ b/resources/tools/testbed-setup/playbooks/03-virl-post-install.yaml @@ -41,6 +41,10 @@ command: ln -sf /home/jenkins-in/testcase-infra/bin/start-testcase /home/jenkins-in/bin/start-testcase args: creates: /home/jenkins-in/bin/start-testcase + - name: Link start-honeycomb-testcase executable + command: ln -sf /home/jenkins-in/testcase-infra/bin/start-honeycomb-testcase /home/jenkins-in/bin/start-honeycomb-testcase + args: + creates: /home/jenkins-in/bin/start-honeycomb-testcase - name: Link stop-testcase executable command: ln -sf /home/jenkins-in/testcase-infra/bin/stop-testcase /home/jenkins-in/bin/stop-testcase args: diff --git a/resources/tools/virl/bin/start-honeycomb-testcase b/resources/tools/virl/bin/start-honeycomb-testcase new file mode 100755 index 0000000000..f38d1a783b --- /dev/null +++ b/resources/tools/virl/bin/start-honeycomb-testcase @@ -0,0 +1,375 @@ +#!/usr/bin/python + +# Copyright (c) 2016 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = 'ckoester@cisco.com' + +import sys +import requests +import re +import os +import argparse +import tempfile +import shutil +import time +import paramiko +import netifaces + +# Commands needed to install VPP and Honeycomb from repository +inst_cmds = [ + 'sudo rm /etc/apt/sources.list.d/99fd.io.list || true', + 'echo "deb [trusted=yes] https://nexus.fd.io/content/repositories/' + 'fd.io.master.ubuntu.trusty.main/ ./" | ' + 'sudo tee -a /etc/apt/sources.list.d/99fd.io.list', + 'sudo apt-get -y update', + 'sudo apt-get -y install vpp vpp-dbg vpp-dev vpp-dpdk-dev vpp-dpdk-dkms ' + 'vpp-lib honeycomb' +] + +# +# Helper function to indent a text string +# +def indent(lines, amount, fillchar=' '): + padding = amount * fillchar + return padding + ('\n'+padding).join(lines.split('\n')) + +def perform_remote_command(ssh_client, cmd, verbosity): + """This function performs a command on the specified remote host using given + ssh client. Depending on the level of verbosity, it prints stdout and + stderr. + + :param ssh_client: SSH client used for communication. + :param cmd: Command to be performed on th rremote client. + :param verbosity: Verbosity level. + :type ssh_client: paramiko.SSHClient + :type cmd: str + :type verbosity: int + """ + + _, stdout, stderr = ssh_client.exec_command(cmd) + c_stdout = stdout.read() + c_stderr = stderr.read() + if verbosity >= 2: + print("DEBUG: Command output was:\n{0}".format(c_stdout)) + print("DEBUG: Command stderr was:\n{0}".format(c_stderr)) + +# +# Main function. +# FIXME: Right now, this is really coded like a shell script, as one big +# function executed in sequence. This should be broken down into multiple +# functions. +# +def main(): + # + # Get our default interface IP address. This will become the default + # value for the "NFS Server IP" option. + # + gws = netifaces.gateways() + addrs = netifaces.ifaddresses(gws['default'][netifaces.AF_INET][1]) + default_addr = addrs[netifaces.AF_INET][0]['addr'] + + # + # Verify CLI parameters and try to download our VPP image into a temporary + # file first + # + parser = argparse.ArgumentParser() + parser.add_argument("topology", help="the base topology to be started") + parser.add_argument("-k", "--keep", help="Keep (do not delete) the " + + "simulation in case of error", action='store_true') + parser.add_argument("-v", "--verbosity", action="count", default=0) + parser.add_argument("-nip", "--nfs-server-ip", help="NFS server (our) IP " + + "default is derived from routing table: " + + "{}".format(default_addr), default=default_addr) + parser.add_argument("-ns", "--nfs-scratch-directory", + help="Server location for NFS scratch diretory", + default="/nfs/scratch") + parser.add_argument("-nc", "--nfs-common-directory", + help="Server location for NFS common (read-only) " + + "directory", default="/nfs/common") + parser.add_argument("-wc", "--wait-count", + help="number of intervals to wait for simulation to " + + "be ready", type=int, default=24) + parser.add_argument("-wt", "--wait-time", + help="length of a single interval to wait for " + + "simulation to be ready", type=int, default=5) + parser.add_argument("-vip", "--virl-ip", + help="VIRL IP and Port (e.g. 127.0.0.1:19399)", + default="127.0.0.1:19399") + parser.add_argument("-u", "--username", help="VIRL username", + default="tb4-virl") + parser.add_argument("-p", "--password", help="VIRL password", + default="Cisco1234") + parser.add_argument("-su", "--ssh-user", help="SSH username", + default="cisco") + parser.add_argument("-spr", "--ssh-privkey", help="SSH private keyfile", + default="/home/jenkins-in/.ssh/id_rsa_virl") + parser.add_argument("-spu", "--ssh-pubkey", help="SSH public keyfile", + default="/home/jenkins-in/.ssh/id_rsa_virl.pub") + parser.add_argument("-r", "--release", help="VM disk image/release " + + "(ex. \"csit-ubuntu-14.04.4_2016-05-25_1.0\")", + default="csit-ubuntu-14.04.4_2016-05-25_1.0") + parser.add_argument("--topology-directory", help="Topology directory", + default="/home/jenkins-in/testcase-infra/topologies") + + args = parser.parse_args() + + # + # Check if topology and template exist + # + if args.verbosity >= 2: + print "DEBUG: Running with topology {}".format(args.topology) + + topology_virl_filename = os.path.join(args.topology_directory, + args.topology + ".virl") + topology_yaml_filename = os.path.join(args.topology_directory, + args.topology + ".yaml") + + if not os.path.isfile(topology_virl_filename): + print "ERROR: Topology VIRL file {} does not exist".\ + format(topology_virl_filename) + sys.exit(1) + if not os.path.isfile(topology_yaml_filename): + print "ERROR: Topology YAML file {} does not exist".\ + format(topology_yaml_filename) + sys.exit(1) + + # + # Start VIRL topology + # + if args.verbosity >= 1: + print "DEBUG: Starting VIRL topology" + temp_handle, temp_topology = tempfile.mkstemp() + with open(args.ssh_pubkey, 'r') as pubkey_file: + pub_key = pubkey_file.read().replace('\n', '') + with open(temp_topology, 'w') as new_file, \ + open(topology_virl_filename, 'r') as old_file: + for line in old_file: + line = line.replace(" - VIRL-USER-SSH-PUBLIC-KEY", " - "+pub_key) + line = line.replace("$$NFS_SERVER_SCRATCH$$", \ + args.nfs_server_ip+":"+args.nfs_scratch_directory) + line = line.replace("$$NFS_SERVER_COMMON$$", \ + args.nfs_server_ip+":"+args.nfs_common_directory) + line = line.replace("$$VM_IMAGE$$", "server-"+args.release) + new_file.write(line) + os.close(temp_handle) + + try: + new_file = open(temp_topology, 'rb') + headers = {'Content-Type': 'text/xml'} + req = requests.post('http://' + args.virl_ip + '/simengine/rest/launch', + headers=headers, + auth=(args.username, args.password), data=new_file) + if args.verbosity >= 2: + print "DEBUG: - Response Code {}".format(req.status_code) + new_file.close() + + except: + print "ERROR: Launching VIRL simulation - received invalid response" + print req + os.remove(temp_topology) + sys.exit(1) + + if req.status_code != 200: + print "ERROR: Launching VIRL simulation - received status other " + \ + "than 200 HTTP OK" + print "Status was: {} \n".format(req.status_code) + print "Response content was: " + print req.content + os.remove(temp_topology) + sys.exit(1) + + # If we got here, we had a good response. The response content is the + # session ID. + session_id = req.content + + # + # Create simulation scratch directory. Move topology file into that + # directory. Copy or move debian packages into that directory. + # + scratch_directory = os.path.join(args.nfs_scratch_directory, session_id) + os.mkdir(scratch_directory) + shutil.move(temp_topology, os.path.join(scratch_directory, + "virl_topology.virl")) + os.mkdir(os.path.join(scratch_directory, "vpp")) + + # + # Wait for simulation to become active + # + if args.verbosity >= 1: + print "DEBUG: Waiting for simulation to become active" + + sim_is_started = False + nodelist = [] + + count = args.wait_count + while (count > 0) and not sim_is_started: + time.sleep(args.wait_time) + count -= 1 + + req = requests.get('http://' + args.virl_ip + '/simengine/rest/nodes/' + + session_id, auth=(args.username, args.password)) + data = req.json() + + active = 0 + total = 0 + + # Flush the node list every time, keep the last one + nodelist = [] + + # Hosts are the keys of the inner dictionary + for key in data[session_id].keys(): + if data[session_id][key]['management-proxy'] == "self": + continue + nodelist.append(key) + total += 1 + if data[session_id][key]['state'] == "ACTIVE": + active += 1 + if args.verbosity >= 2: + print "DEBUG: - Attempt {} out of {}, total {} hosts, {} active".\ + format(args.wait_count-count, args.wait_count, total, active) + if active == total: + sim_is_started = True + + if not sim_is_started: + print "ERROR: Simulation started OK but devices never changed to " + \ + "ACTIVE state" + print "Last VIRL response:" + print data + if not args.keep: + shutil.rmtree(scratch_directory) + req = requests.get('http://' + args.virl_ip + + '/simengine/rest/stop/' + session_id, + auth=(args.username, args.password)) + + if args.verbosity >= 2: + print "DEBUG: Nodes: " + ", ".join(nodelist) + + # + # Fetch simulation's IPs and create files + # (ansible hosts file, topology YAML file) + # + req = requests.get('http://' + args.virl_ip + + '/simengine/rest/interfaces/' + session_id + + '?fetch-state=1', auth=(args.username, args.password)) + data = req.json() + + # Populate node addresses + nodeaddrs = {} + topology = {} + for key in nodelist: + nodetype = re.split('[0-9]', key)[0] + if not nodetype in nodeaddrs: + nodeaddrs[nodetype] = {} + nodeaddrs[nodetype][key] = re.split('\\/', \ + data[session_id][key]['management']['ip-address'])[0] + if args.verbosity >= 2: + print "DEBUG: Node {} is of type {} and has management IP {}".\ + format(key, nodetype, nodeaddrs[nodetype][key]) + + topology[key] = {} + for key2 in data[session_id][key]: + topology[key]["nic-"+key2] = data[session_id][key][key2] + if 'ip-address' in topology[key]["nic-"+key2]: + if topology[key]["nic-"+key2]['ip-address'] is not None: + topology[key]["nic-"+key2]['ip-addr'] = re.split('\\/', \ + topology[key]["nic-"+key2]['ip-address'])[0] + + # Write ansible file + ansiblehosts = open(os.path.join(scratch_directory, 'ansible-hosts'), 'w') + for key1 in nodeaddrs: + ansiblehosts.write("[{}]\n".format(key1)) + for key2 in nodeaddrs[key1]: + ansiblehosts.write("{} hostname={}\n".format(nodeaddrs[key1][key2], + key2)) + ansiblehosts.close() + + # Process topology YAML template + with open(args.ssh_privkey, 'r') as privkey_file: + priv_key = indent(privkey_file.read(), 6) + + with open(os.path.join(scratch_directory, "topology.yaml"), 'w') as \ + new_file, open(topology_yaml_filename, 'r') as old_file: + for line in old_file: + new_file.write(line.format(priv_key=priv_key, topology=topology)) + + # + # Wait for hosts to become reachable over SSH + # + if args.verbosity >= 1: + print "DEBUG: Waiting for hosts to become reachable using SSH" + + missing = -1 + count = args.wait_count + while (count > 0) and missing != 0: + time.sleep(args.wait_time) + count -= 1 + + missing = 0 + for key in nodelist: + if not os.path.exists(os.path.join(scratch_directory, key)): + missing += 1 + if args.verbosity >= 2: + print "DEBUG: - Attempt {} out of {}, waiting for {} hosts".\ + format(args.wait_count-count, args.wait_count, missing) + + if missing != 0: + print "ERROR: Simulation started OK but {} hosts ".format(missing) + \ + "never mounted their NFS directory" + if not args.keep: + shutil.rmtree(scratch_directory) + req = requests.get('http://' + args.virl_ip + + '/simengine/rest/stop/' + session_id, + auth=(args.username, args.password)) + + # + # Upgrade VPP + # + if args.verbosity >= 1: + print("DEBUG: Uprading VPP") + + for key1 in nodeaddrs: + if not key1 == 'tg': + for key2 in nodeaddrs[key1]: + ipaddr = nodeaddrs[key1][key2] + if args.verbosity >= 2: + print("DEBUG: Upgrading VPP on node {}".format(ipaddr)) + paramiko.util.log_to_file(os.path.join(scratch_directory, + "ssh.log")) + client = paramiko.SSHClient() + client.load_system_host_keys() + client.load_host_keys("/dev/null") + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect(ipaddr, username=args.ssh_user, + key_filename=args.ssh_privkey) + for cmd in inst_cmds: + perform_remote_command(client, cmd, args.verbosity) + + # + # Write a file with timestamp to scratch directory. We can use this to track + # how long a simulation has been running. + # + with open(os.path.join(scratch_directory, 'start_time'), 'a') as \ + timestampfile: + timestampfile.write('{}\n'.format(int(time.time()))) + + # + # Declare victory + # + if args.verbosity >= 1: + print "SESSION ID: {}".format(session_id) + + print "{}".format(session_id) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/virl_params.sh b/virl_params.sh new file mode 100644 index 0000000000..e36058d06d --- /dev/null +++ b/virl_params.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) 2016 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# VIRL server parameters used by bootstrap scrips: +VIRL_SERVERS=("10.30.51.28" "10.30.51.29" "10.30.51.30") +VIRL_SERVER="" + +VIRL_USERNAME=jenkins-in +VIRL_PKEY=priv_key +VIRL_SERVER_STATUS_FILE="status" +VIRL_SERVER_EXPECTED_STATUS="PRODUCTION" |