aboutsummaryrefslogtreecommitdiffstats
path: root/.gitignore
blob: 3e9a92e44ac4e6c640adea434d1e5deef2517ec5 (plain) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

*~
*.sw[op]

/build-root/.ccache
/build-root/build-*/
/build-root/install-*/
/build-root/tools
/build-root/dpdk-includes/
/build-root/packages-vpp/
/build-root/path_setup
/build-root/deb/debian/vpp-plugins/
/build-root/deb/.pybuild/
/build-root/*.deb
/build-root/*.rpm
/build-root/*.tar.xz
/build-root/*.changes
/build-root/rpmbuild/
/build-root/test-doc/
/build-root/test-cov/
/test/run/
/test/build/
/test/coverage/
/build-config.mk
/build/external/*.tar.gz
/build/external/*.tar.xz
/build/external/vpp-*.deb
/build/external/vpp-*.changes
/build/external/downloads/
/path_setup
/tools/
# start autotools ignore
autom4te.cache/
config/
Makefile.in
aclocal.m4
app.info
compile
config.log
config.guess
config.sub
config.status
configure
configure.scan
coverage_report
depcomp
install-sh
libtool
m4
missing
stamp-h1
ltmain.sh
ylwrap
test-driver

*.iml
.deps.ok
.settings
.autotools
# stop autotools ignore

# OSX and some IDE
.DS_Store
.idea/
.project
.cproject
.pydevproject
.vscode/
cmake-build*/

# cscope and ctags
/cscope.*
/tags
ID
TAGS
# ggtags
GPATH
GRTAGS
GTAGS
# Generated documentation
/build-root/docs
/build-root/.doxygen-bootstrap.ok
/build-root/.doxygen-siphon.dep
/docs/_build
/sphinx_venv
!/docs/Makefile

# indent backup files
*.BAK

# Python bytecode
*.pyc

# Python api generator
/src/vpp-api/python/build
/src/vpp-api/python/dist
/src/vpp-api/python/vpp_papi.egg-info
/src/vpp_api/.tox
/src/vpp_api/.stestr

# vppctl command list
/src/scripts/vppctl-cmd-list

# No core files
**/core

# extra scripts config
/extras/scripts/.config/

# extras gomemif build files
/extras/gomemif/bazel*
ighlight .c1 { color: #75715e } /* Comment.Single */ .highlight .cs { color: #75715e } /* Comment.Special */ .highlight .gd { color: #f92672 } /* Generic.Deleted */ .highlight .ge { font-style: italic } /* Generic.Emph */ .highlight .gi { color: #a6e22e } /* Generic.Inserted */ .highlight .gs { font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #75715e } /* Generic.Subheading */ .highlight .kc { color: #66d9ef } /* Keyword.Constant */ .highlight .kd { color: #66d9ef } /* Keyword.Declaration */ .highlight .kn { color: #f92672 } /* Keyword.Namespace */ .highlight .kp { color: #66d9ef } /* Keyword.Pseudo */ .highlight .kr { color: #66d9ef } /* Keyword.Reserved */ .highlight .kt { color: #66d9ef } /* Keyword.Type */ .highlight .ld { color: #e6db74 } /* Literal.Date */ .highlight .m { color: #ae81ff } /* Literal.Number */ .highlight .s { color: #e6db74 } /* Literal.String */ .highlight .na { color: #a6e22e } /* Name.Attribute */ .highlight .nb { color: #f8f8f2 } /* Name.Builtin */ .highlight .nc { color: #a6e22e } /* Name.Class */ .highlight .no { color: #66d9ef } /* Name.Constant */ .highlight .nd { color: #a6e22e } /* Name.Decorator */ .highlight .ni { color: #f8f8f2 } /* Name.Entity */ .highlight .ne { color: #a6e22e } /* Name.Exception */ .highlight .nf { color: #a6e22e } /* Name.Function */ .highlight .nl { color: #f8f8f2 } /* Name.Label */ .highlight .nn { color: #f8f8f2 } /* Name.Namespace */ .highlight .nx { color: #a6e22e } /* Name.Other */ .highlight .py { color: #f8f8f2 } /* Name.Property */ .highlight .nt { color: #f92672 } /* Name.Tag */ .highlight .nv { color: #f8f8f2 } /* Name.Variable */ .highlight .ow { color: #f92672 } /* Operator.Word */ .highlight .w { color: #f8f8f2 } /* Text.Whitespace */ .highlight .mb { color: #ae81ff } /* Literal.Number.Bin */ .highlight .mf { color: #ae81ff } /* Literal.Number.Float */ .highlight .mh { color: #ae81ff } /* Literal.Number.Hex */ .highlight .mi { color: #ae81ff } /* Literal.Number.Integer */ .highlight .mo { color: #ae81ff } /* Literal.Number.Oct */ .highlight .sa { color: #e6db74 } /* Literal.String.Affix */ .highlight .sb { color: #e6db74 } /* Literal.String.Backtick */ .highlight .sc { color: #e6db74 } /* Literal.String.Char */ .highlight .dl { color: #e6db74 } /* Literal.String.Delimiter */ .highlight .sd { color: #e6db74 } /* Literal.String.Doc */ .highlight .s2 { color: #e6db74 } /* Literal.String.Double */ .highlight .se { color: #ae81ff } /* Literal.String.Escape */ .highlight .sh { color: #e6db74 } /* Literal.String.Heredoc */ .highlight .si { color: #e6db74 } /* Literal.String.Interpol */ .highlight .sx { color: #e6db74 } /* Literal.String.Other */ .highlight .sr { color: #e6db74 } /* Literal.String.Regex */ .highlight .s1 { color: #e6db74 } /* Literal.String.Single */ .highlight .ss { color: #e6db74 } /* Literal.String.Symbol */ .highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #a6e22e } /* Name.Function.Magic */ .highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */ .highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */ .highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */ .highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */ .highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */ } @media (prefers-color-scheme: light) { .highlight .hll { background-color: #ffffcc } .highlight .c { color: #888888 } /* Comment */ .highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ .highlight .k { color: #008800; font-weight: bold } /* Keyword */ .highlight .ch { color: #888888 } /* Comment.Hashbang */ .highlight .cm { color: #888888 } /* Comment.Multiline */ .highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */ .highlight .cpf { color: #888888 } /* Comment.PreprocFile */ .highlight .c1 { color: #888888 } /* Comment.Single */ .highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */ .highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ .highlight .ge { font-style: italic } /* Generic.Emph */ .highlight .gr { color: #aa0000 } /* Generic.Error */ .highlight .gh { color: #333333 } /* Generic.Heading */ .highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ .highlight .go { color: #888888 } /* Generic.Output */ .highlight .gp { color: #555555 } /* Generic.Prompt */ .highlight .gs { font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #666666 } /* Generic.Subheading */ .highlight .gt { color: #aa0000 } /* Generic.Traceback */ .highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008800 } /* Keyword.Pseudo */ .highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ } 
#!/usr/bin/env python

# Copyright (c) 2016 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This is a helper script to make test execution easy."""

from __future__ import print_function
import sys
import os
import time
from string import ascii_lowercase
from random import sample
import argparse
from pykwalify.core import Core
from pykwalify.errors import PyKwalifyException
from yaml import load
import robot
from robot.errors import DATA_ERROR, DataError, FRAMEWORK_ERROR, FrameworkError
from robot.run import RobotFramework
from robot.conf.settings import RobotSettings
from robot.running.builder import TestSuiteBuilder
from robot.running.model import TestSuite

TOPOLOGIES_DIR = './topologies/enabled/'
TESTS_DIR = './tests'
OUTPUTS_DIR = './outputs'


def get_suite_list(*datasources, **options):
    """Returns filtered test suites based on include exclude tags

    :param datasources: paths to tests
    :param options: Robot Framework options (robot.conf.settings.py)
    :return: list of Robot Framework TestSuites which contain tests
    """
    class _MyRobotFramework(RobotFramework):
        """Custom implementation of RobotFramework main()."""
        def main(self, datasources, **options):
            # copied from robot.run.RobotFramework.main
            settings = RobotSettings(options)
            test_suite = TestSuiteBuilder(settings['SuiteNames'],
                                          settings['WarnOnSkipped'])
            # pylint: disable=star-args
            suite = test_suite.build(*datasources)
            suite.configure(**settings.suite_config)

            return suite

    # get all test cases list without run tests, execute runs overloaded main
    # function
    suite = _MyRobotFramework().execute(*datasources, output=None, dryrun=True,
                                        **options)
    if isinstance(suite, TestSuite):
        suites = []
        suites.append(suite)
        append_new = True
        while append_new:
            append_new = False
            tmp = []
            for suite in suites:
                # pylint: disable=protected-access
                if len(suite.suites._items) > 0:
                    for i in suite.suites._items:
                        tmp.append(i)
                    append_new = True
                else:
                    tmp.append(suite)
            suites = tmp
        return suites
        # TODO: check testcases Tags ? all tests should have same set of tags
    else:
        if suite == DATA_ERROR:
            raise DataError
        if suite == FRAMEWORK_ERROR:
            raise FrameworkError
        return []


def run_suites(tests_dir, suites, output_dir, output_prefix='suite',
               **options):
    """Execute RF's run with parameters."""

    with open('{}/{}.out'.format(output_dir, output_prefix), 'w') as out:
        robot.run(tests_dir,
                  suite=[s.longname for s in suites],
                  output='{}/{}.xml'.format(output_dir, output_prefix),
                  debugfile='{}/{}.log'.format(output_dir, output_prefix),
                  log=None,
                  report=None,
                  stdout=out,
                  **options)


def parse_outputs(output_dir):
    """Parse output xmls from all executed tests."""

    outs = [os.path.join(output_dir, file_name)
            for file_name in os.listdir(output_dir)
            if file_name.endswith('.xml')]
    # pylint: disable=star-args
    robot.rebot(*outs, merge=True)


def topology_lookup(topology_paths, topo_dir, validate):
    """Make topology list and validate topologies against schema

    :param parsed_args: topology list, is empty then scans topologies in
                        topo_dir
    :param topo_dir: scan directory for topologies
    :param validate: if True then validate topology
    :return: list of topologies
    """

    ret_topologies = []
    if topology_paths:
        for topo in topology_paths:
            if os.path.exists(topo):
                ret_topologies.append(topo)
            else:
                print("Topology file {} doesn't exist".format(topo),
                      file=sys.stderr)
    else:
        ret_topologies = [os.path.join(topo_dir, file_name)
                          for file_name in os.listdir(topo_dir)
                          if file_name.lower().endswith('.yaml')]

    if len(ret_topologies) == 0:
        print('No valid topology found', file=sys.stderr)
        exit(1)

    # validate topologies against schema
    exit_on_error = False
    for topology_name in ret_topologies:
        try:
            with open(topology_name) as file_name:
                yaml_obj = load(file_name)
            core = Core(source_file=topology_name,
                        schema_files=yaml_obj["metadata"]["schema"])
            core.validate()
        except PyKwalifyException as ex:
            print('Unable to verify topology {}, schema error: {}'.\
                  format(topology_name, ex),
                  file=sys.stderr)
            exit_on_error = True
        except KeyError as ex:
            print('Unable to verify topology {}, key error: {}'.\
                  format(topology_name, ex),
                  file=sys.stderr)
            exit_on_error = True
        except Exception as ex:
            print('Unable to verify topology {}, {}'.format(topology_name, ex),
                  file=sys.stderr)
            exit_on_error = True

    if exit_on_error and validate:
        exit(1)

    return ret_topologies


def main():
    """Main function."""
    parser = argparse.ArgumentParser(description='A test runner')
    parser.add_argument('-i', '--include', action='append',
                        help='include tests with tag')
    parser.add_argument('-e', '--exclude', action='append',
                        help='exclude tests with tag')
    parser.add_argument('-s', '--suite', action='append',
                        help='full name of suite to run')
    parser.add_argument('-t', '--topology', action='append',
                        help='topology where tests should be run')
    parser.add_argument('-d', '--test_dir', nargs='?', default=TESTS_DIR,
                        help='where tests are stored')
    parser.add_argument('-o', '--output_dir', nargs='?', default=OUTPUTS_DIR,
                        help='where results are stored')
    parser.add_argument('-L', '--loglevel', nargs='?', default='INFO', type=str,
                        choices=['TRACE', 'DEBUG', 'INFO', 'WARN', 'NONE'],
                        help='robot frameworks level for logging')
    parser.add_argument('-n', '--no_validate', action="store_false",
                        help='Do not exit if topology validation failed')

    args = parser.parse_args()

    i = args.include or []
    excl = args.exclude or []
    suite_filter = args.suite or []
    test_dir = args.test_dir

    # prepare output subdir
    suite_output_dir = os.path.join(args.output_dir,
                                    time.strftime('%y%m%d%H%M%S'))
    os.makedirs(suite_output_dir)

    topologies = topology_lookup(args.topology, TOPOLOGIES_DIR,
                                 args.no_validate)
    suite_list = get_suite_list(test_dir, include=i, exclude=excl,
                                suite=suite_filter)

    # TODO: do the topology suite mapping magic
    #       for now all tests on single topology
    if len(topologies) > 1:
        print('Multiple topologies unsupported yet', file=sys.stderr)
        exit(1)
    topology_suite_mapping = {topologies[0]: suite_list}

    # on all topologies, run test
    # TODO: run parallel
    for topology_path, topology_suite_list in topology_suite_mapping.items():
        topology_path_variable = 'TOPOLOGY_PATH:{}'.format(topology_path)
        variables = [topology_path_variable]
        print('Runing tests on topology {}'.format(topology_path))
        run_suites(test_dir, topology_suite_list, variable=variables,
                   output_dir=suite_output_dir,
                   output_prefix=''.join(sample(ascii_lowercase, 5)),
                   include=i, exclude=excl, loglevel=args.loglevel)

    print('Parsing test results')
    parse_outputs(suite_output_dir)


if __name__ == "__main__":
    main()