From e2f9bdd098fb0008cc4fc2f8c8298ebe0d6ef062 Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Fri, 19 May 2017 09:42:39 +0200 Subject: CSIT-607 Optimize VIRL job scheduling algorithm Optimize VIRL job scheduling algorithm based on available VIRL host capacity. Add IP pool availability pre-checks. - add quota based on max. allowe IPs and max. allowe VMs per virl - use one common bootstrap file instead of two separate files (one for ubutnut, another for centos) Change-Id: Ic40122a084624ff9c5eafa9f372b0451e857e29a Signed-off-by: Peter Mikus --- resources/tools/virl/bin/list-testcases | 19 ++++++++ resources/tools/virl/bin/start-testcase | 85 +++++++++++++++++++++++++++++---- 2 files changed, 95 insertions(+), 9 deletions(-) create mode 100755 resources/tools/virl/bin/list-testcases (limited to 'resources/tools/virl') diff --git a/resources/tools/virl/bin/list-testcases b/resources/tools/virl/bin/list-testcases new file mode 100755 index 0000000000..73c05303fe --- /dev/null +++ b/resources/tools/virl/bin/list-testcases @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright (c) 2017 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +VIRL_USER="tb4-virl" # VIRL credentials (what one would enter in VMMaestro) +VIRL_PASSWORD="Cisco1234" + +virl_std_client --json --quiet -u $VIRL_USER -p $VIRL_PASSWORD simengine-list diff --git a/resources/tools/virl/bin/start-testcase b/resources/tools/virl/bin/start-testcase index 0d9c49e049..f6c3cc32b8 100755 --- a/resources/tools/virl/bin/start-testcase +++ b/resources/tools/virl/bin/start-testcase @@ -17,18 +17,21 @@ __author__ = 'ckoester@cisco.com' -import sys -import re -import os import argparse -import tempfile +import netifaces +import os +import paramiko +import random +import re import shutil +import sys +import tempfile import time -import paramiko -import netifaces import requests +IPS_PER_SIMULATION = 5 + def indent(lines, amount, fillchar=' '): """Indent the string by amount of fill chars. @@ -57,6 +60,66 @@ def print_to_stderr(msg, end='\n'): except ValueError: pass +def get_assigned_interfaces(args, network="flat"): + """Retrieve assigned interfaces in openstack network. + + :param args: Command line params. + :param network: Openstack network. + :type args: ArgumentParser + :type network: str + :returns: Assigned interfaces. + :rtype: list + :raises RuntimeError: If response is not 200. + """ + req = requests.get('http://{}/openstack/rest/ports/{}' + .format(args.virl_ip, network), + auth=(args.username, args.password)) + if req.status_code == 200: + return req.json() + else: + raise RuntimeError("ERROR: Retrieving ports in use - " + "Status other than 200 HTTP OK:\n{}" + .format(req.content)) + +def get_assigned_interfaces_count(args, network="flat"): + """Count assigned interfaces in openstack network. + + :param args: Command line params. + :param network: Openstack network. + :type args: ArgumentParser + :type network: str + :returns: Assigned interfaces count. + :rtype: int + """ + return len(get_assigned_interfaces(args, network=network)) + +def check_ip_addresses(args): + """Check IP address availability. + + :param args: Command line params. + :type args: ArgumentParser + :raises RuntimeError: If not enough free addresses available. + """ + for i in range(args.wait_count): + if (args.quota - \ + get_assigned_interfaces_count(args) >= IPS_PER_SIMULATION): + break + if args.verbosity >= 2: + print_to_stderr("DEBUG: - Attempt {} out of {}, waiting for free " + "IP addresses".format(i, args.wait_count)) + # Wait random amount of time within range 1-3 minutes + time.sleep(random.randint(60,180)) + else: + raise RuntimeError("ERROR: Not enough IP addresses to run simulation") + +def check_virl_resources(args): + """Check virl resources availability. + + :param args: Command line params. + :type args: ArgumentParser + """ + check_ip_addresses(args) + # # FIXME: Right now, this is really coded like a shell script, as one big # function executed in sequence. This should be broken down into multiple @@ -123,6 +186,9 @@ def main(): default="csit-ubuntu-16.04.1_2016-12-19_1.6") parser.add_argument("--topology-directory", help="Topology directory", default="/home/jenkins-in/testcase-infra/topologies") + parser.add_argument("-q", "--quota", + help="VIRL quota for max number of allowed IPs", + type=int, default=74) args = parser.parse_args() @@ -181,6 +247,7 @@ def main(): try: data = open(temp_topology, 'rb') + check_virl_resources(args) req = requests.post('http://' + args.virl_ip + '/simengine/rest/launch', auth=(args.username, args.password), data=data) @@ -310,7 +377,7 @@ def main(): shutil.rmtree(scratch_directory) except shutil.Error: print_to_stderr("ERROR: Removing scratch directory") - print "{}".format(session_id) + print "{}".format(session_id) sys.exit(1) if args.verbosity >= 2: @@ -348,7 +415,7 @@ def main(): shutil.rmtree(scratch_directory) except shutil.Error: print_to_stderr("ERROR: Removing scratch directory") - print "{}".format(session_id) + print "{}".format(session_id) sys.exit(1) data = req.json() @@ -423,7 +490,7 @@ def main(): shutil.rmtree(scratch_directory) except shutil.Error: print_to_stderr("ERROR: Removing scratch directory") - print "{}".format(session_id) + print "{}".format(session_id) sys.exit(1) # -- cgit 1.2.3-korg