#!/usr/bin/env python
# Copyright (c) 2016 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This is a helper script to make test execution easy."""
from __future__ import print_function
import sys
import os
import time
from string import ascii_lowercase
from random import sample
import argparse
from pykwalify.core import Core
from pykwalify.errors import PyKwalifyException
from yaml import load
import robot
from robot.errors import DATA_ERROR, DataError, FRAMEWORK_ERROR, FrameworkError
from robot.run import RobotFramework
from robot.conf.settings import RobotSettings
from robot.running.builder import TestSuiteBuilder
from robot.running.model import TestSuite
TOPOLOGIES_DIR = './topologies/enabled/'
TESTS_DIR = './tests'
OUTPUTS_DIR = './outputs'
def get_suite_list(*datasources, **options):
"""Returns filtered test suites based on include exclude tags
:param datasources: paths to tests
:param options: Robot Framework options (robot.conf.settings.py)
:return: list of Robot Framework TestSuites which contain tests
"""
class _MyRobotFramework(RobotFramework):
"""Custom implementation of RobotFramework main()."""
def main(self, datasources, **options):
# copied from robot.run.RobotFramework.main
settings = RobotSettings(options)
test_suite = TestSuiteBuilder(settings['SuiteNames'],
settings['WarnOnSkipped'])
# pylint: disable=star-args
suite = test_suite.build(*datasources)
suite.configure(**settings.suite_config)
return suite
# get all test cases list without run tests, execute runs overloaded main
# function
suite = _MyRobotFramework().execute(*datasources, output=None, dryrun=True,
**options)
if isinstance(suite, TestSuite):
suites = []
suites.append(suite)
append_new = True
while append_new:
append_new = False
tmp = []
for suite in suites:
# pylint: disable=protected-access
if len(suite.suites._items) > 0:
for i in suite.suites._items:
tmp.append(i)
append_new = True
else:
tmp.append(suite)
suites = tmp
return suites
# TODO: check testcases Tags ? all tests should have same set of tags
else:
if suite == DATA_ERROR:
raise DataError
if suite == FRAMEWORK_ERROR:
raise FrameworkError
return []
def run_suites(tests_dir, suites, output_dir, output_prefix='suite',
**options):
"""Execute RF's run with parameters."""
with open('{}/{}.out'.format(output_dir, output_prefix), 'w') as out:
robot.run(tests_dir,
suite=[s.longname for s in suites],
output='{}/{}.xml'.format(output_dir, output_prefix),
debugfile='{}/{}.log'.format(output_dir, output_prefix),
log=None,
report=None,
stdout=out,
**options)
def parse_outputs(output_dir):
"""Parse output xmls from all executed tests."""
outs = [os.path.join(output_dir, file_name)
for file_name in os.listdir(output_dir)
if file_name.endswith('.xml')]
# pylint: disable=star-args
robot.rebot(*outs, merge=True)
def topology_lookup(topology_paths, topo_dir, validate):
"""Make topology list and validate topologies against schema
:param parsed_args: topology list, is empty then scans topologies in
topo_dir
:param topo_dir: scan directory for topologies
:param validate: if True then validate topology
:return: list of topologies
"""
ret_topologies = []
if topology_paths:
for topo in topology_paths:
if os.path.exists(topo):
ret_topologies.append(topo)
else:
print("Topology file {} doesn't exist".format(topo),
file=sys.stderr)
else:
ret_topologies = [os.path.join(topo_dir, file_name)
for file_name in os.listdir(topo_dir)
if file_name.lower().endswith('.yaml')]
if len(ret_topologies) == 0:
print('No valid topology found', file=sys.stderr)
exit(1)
# validate topologies against schema
exit_on_error = False
for topology_name in ret_topologies:
try:
with open(topology_name) as file_name:
yaml_obj = load(file_name)
core = Core(source_file=topology_name,
schema_files=yaml_obj["metadata"]["schema"])
core.validate()
except PyKwalifyException as ex:
print('Unable to verify topology {}, schema error: {}'.\
format(topology_name, ex),
file=sys.stderr)
exit_on_error = True
except KeyError as ex:
print('Unable to verify topology {}, key error: {}'.\
format(topology_name, ex),
file=sys.stderr)
exit_on_error = True
except Exception as ex:
print('Unable to verify topology {}, {}'.format(topology_name, ex),
file=sys.stderr)
exit_on_error = True
if exit_on_error and validate:
exit(1)
return ret_topologies
def main():
"""Main function."""
parser = argparse.ArgumentParser(description='A test runner')
parser.add_argument('-i', '--include', action='append',
help='include tests with tag')
parser.add_argument('-e', '--exclude', action='append',
help='exclude tests with tag')
parser.add_argument('-s', '--suite', action='append',
help='full name of suite to run')
parser.add_argument('-t', '--topology', action='append',
help='topology where tests should be run')
parser.add_argument('-d', '--test_dir', nargs='?', default=TESTS_DIR,
help='where tests are stored')
parser.add_argument('-o', '--output_dir', nargs='?', default=OUTPUTS_DIR,
help='where results are stored')
parser.add_argument('-L', '--loglevel', nargs='?', default='INFO', type=str,
choices=['TRACE', 'DEBUG', 'INFO', 'WARN', 'NONE'],
help='robot frameworks level for logging')
parser.add_argument('-n', '--no_validate', action="store_false",
help='Do not exit if topology validation failed')
args = parser.parse_args()
i = args.include or []
excl = args.exclude or []
suite_filter = args.suite or []
test_dir = args.test_dir
# prepare output subdir
suite_output_dir = os.path.join(args.output_dir,
time.strftime('%y%m%d%H%M%S'))
os.makedirs(suite_output_dir)
topologies = topology_lookup(args.topology, TOPOLOGIES_DIR,
args.no_validate)
suite_list = get_suite_list(test_dir, include=i, exclude=excl,
suite=suite_filter)
# TODO: do the topology suite mapping magic
# for now all tests on single topology
if len(topologies) > 1:
print('Multiple topologies unsupported yet', file=sys.stderr)
exit(1)
topology_suite_mapping = {topologies[0]: suite_list}
# on all topologies, run test
# TODO: run parallel
for topology_path, topology_suite_list in topology_suite_mapping.items():
topology_path_variable = 'TOPOLOGY_PATH:{}'.format(topology_path)
variables = [topology_path_variable]
print('Runing tests on topology {}'.format(topology_path))
run_suites(test_dir, topology_suite_list, variable=variables,
output_dir=suite_output_dir,
output_prefix=''.join(sample(ascii_lowercase, 5)),
include=i, exclude=excl, loglevel=args.loglevel)
print('Parsing test results')
parse_outputs(suite_output_dir)
if __name__ == "__main__":
main()