From 33499c81c94c2d3baef9d3e9f061cd76ef86fa74 Mon Sep 17 00:00:00 2001 From: Stefan Kobza Date: Mon, 11 Jan 2016 18:03:25 +0100 Subject: New version of RF tests. Change-Id: I241a2b7a7706e65f71cfd4a62e2a40f053fc5d07 Signed-off-by: Stefan Kobza --- main.py | 233 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100755 main.py (limited to 'main.py') diff --git a/main.py b/main.py new file mode 100755 index 0000000000..d575567854 --- /dev/null +++ b/main.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This is a helper script to make test execution easy.""" + +from __future__ import print_function +import sys +import os +import time +from string import ascii_lowercase +from random import sample +import argparse +from pykwalify.core import Core +from pykwalify.errors import PyKwalifyException +from yaml import load +import robot +from robot.errors import DATA_ERROR, DataError, FRAMEWORK_ERROR, FrameworkError +from robot.run import RobotFramework +from robot.conf.settings import RobotSettings +from robot.running.builder import TestSuiteBuilder +from robot.running.model import TestSuite + +TOPOLOGIES_DIR = './topologies/enabled/' +TESTS_DIR = './tests' +OUTPUTS_DIR = './outputs' + + +def get_suite_list(*datasources, **options): + """Returns filtered test suites based on include exclude tags + + :param datasources: paths to tests + :param options: Robot Framework options (robot.conf.settings.py) + :return: list of Robot Framework TestSuites which contain tests + """ + class _MyRobotFramework(RobotFramework): + """Custom implementation of RobotFramework main().""" + def main(self, datasources, **options): + # copied from robot.run.RobotFramework.main + settings = RobotSettings(options) + test_suite = TestSuiteBuilder(settings['SuiteNames'], + settings['WarnOnSkipped']) + # pylint: disable=star-args + suite = test_suite.build(*datasources) + suite.configure(**settings.suite_config) + + return suite + + # get all test cases list without run tests, execute runs overloaded main + # function + suite = _MyRobotFramework().execute(*datasources, output=None, dryrun=True, + **options) + if isinstance(suite, TestSuite): + suites = [] + suites.append(suite) + append_new = True + while append_new: + append_new = False + tmp = [] + for suite in suites: + # pylint: disable=protected-access + if len(suite.suites._items) > 0: + for i in suite.suites._items: + tmp.append(i) + append_new = True + else: + tmp.append(suite) + suites = tmp + return suites + # TODO: check testcases Tags ? all tests should have same set of tags + else: + if suite == DATA_ERROR: + raise DataError + if suite == FRAMEWORK_ERROR: + raise FrameworkError + return [] + + +def run_suites(tests_dir, suites, output_dir, output_prefix='suite', + **options): + """Execute RF's run with parameters.""" + + with open('{}/{}.out'.format(output_dir, output_prefix), 'w') as out: + robot.run(tests_dir, + suite=[s.longname for s in suites], + output='{}/{}.xml'.format(output_dir, output_prefix), + debugfile='{}/{}.log'.format(output_dir, output_prefix), + log=None, + report=None, + stdout=out, + **options) + + +def parse_outputs(output_dir): + """Parse output xmls from all executed tests.""" + + outs = [os.path.join(output_dir, file_name) + for file_name in os.listdir(output_dir) + if file_name.endswith('.xml')] + # pylint: disable=star-args + robot.rebot(*outs, merge=True) + + +def topology_lookup(topology_paths, topo_dir, validate): + """Make topology list and validate topologies against schema + + :param parsed_args: topology list, is empty then scans topologies in + topo_dir + :param topo_dir: scan directory for topologies + :param validate: if True then validate topology + :return: list of topologies + """ + + ret_topologies = [] + if topology_paths: + for topo in topology_paths: + if os.path.exists(topo): + ret_topologies.append(topo) + else: + print("Topology file {} doesn't exist".format(topo), + file=sys.stderr) + else: + ret_topologies = [os.path.join(topo_dir, file_name) + for file_name in os.listdir(topo_dir) + if file_name.lower().endswith('.yaml')] + + if len(ret_topologies) == 0: + print('No valid topology found', file=sys.stderr) + exit(1) + + # validate topologies against schema + exit_on_error = False + for topology_name in ret_topologies: + try: + with open(topology_name) as file_name: + yaml_obj = load(file_name) + core = Core(source_file=topology_name, + schema_files=yaml_obj["metadata"]["schema"]) + core.validate() + except PyKwalifyException as ex: + print('Unable to verify topology {}, schema error: {}'.\ + format(topology_name, ex), + file=sys.stderr) + exit_on_error = True + except KeyError as ex: + print('Unable to verify topology {}, key error: {}'.\ + format(topology_name, ex), + file=sys.stderr) + exit_on_error = True + except Exception as ex: + print('Unable to verify topology {}, {}'.format(topology_name, ex), + file=sys.stderr) + exit_on_error = True + + if exit_on_error and validate: + exit(1) + + return ret_topologies + + +def main(): + """Main function.""" + parser = argparse.ArgumentParser(description='A test runner') + parser.add_argument('-i', '--include', action='append', + help='include tests with tag') + parser.add_argument('-e', '--exclude', action='append', + help='exclude tests with tag') + parser.add_argument('-s', '--suite', action='append', + help='full name of suite to run') + parser.add_argument('-t', '--topology', action='append', + help='topology where tests should be run') + parser.add_argument('-d', '--test_dir', nargs='?', default=TESTS_DIR, + help='where tests are stored') + parser.add_argument('-o', '--output_dir', nargs='?', default=OUTPUTS_DIR, + help='where results are stored') + parser.add_argument('-L', '--loglevel', nargs='?', default='INFO', type=str, + choices=['TRACE', 'DEBUG', 'INFO', 'WARN', 'NONE'], + help='robot frameworks level for logging') + parser.add_argument('-n', '--no_validate', action="store_false", + help='Do not exit if topology validation failed') + + args = parser.parse_args() + + i = args.include or [] + excl = args.exclude or [] + suite_filter = args.suite or [] + test_dir = args.test_dir + + # prepare output subdir + suite_output_dir = os.path.join(args.output_dir, + time.strftime('%y%m%d%H%M%S')) + os.makedirs(suite_output_dir) + + topologies = topology_lookup(args.topology, TOPOLOGIES_DIR, + args.no_validate) + suite_list = get_suite_list(test_dir, include=i, exclude=excl, + suite=suite_filter) + + # TODO: do the topology suite mapping magic + # for now all tests on single topology + if len(topologies) > 1: + print('Multiple topologies unsupported yet', file=sys.stderr) + exit(1) + topology_suite_mapping = {topologies[0]: suite_list} + + # on all topologies, run test + # TODO: run parallel + for topology_path, topology_suite_list in topology_suite_mapping.items(): + topology_path_variable = 'TOPOLOGY_PATH:{}'.format(topology_path) + variables = [topology_path_variable] + print('Runing tests on topology {}'.format(topology_path)) + run_suites(test_dir, topology_suite_list, variable=variables, + output_dir=suite_output_dir, + output_prefix=''.join(sample(ascii_lowercase, 5)), + include=i, exclude=excl, loglevel=args.loglevel) + + print('Parsing test results') + parse_outputs(suite_output_dir) + + +if __name__ == "__main__": + main() -- cgit 1.2.3-korg