diff options
author | 2021-12-15 17:14:36 +0100 | |
---|---|---|
committer | 2021-12-15 17:14:36 +0100 | |
commit | 01d8f262afc567c3d49a23c3cb2cdeaced8a6887 (patch) | |
tree | 0449c972d8201be16d648dd749e0a7d116aa8b71 /resources/libraries/python/model/ExportResult.py | |
parent | cca05a55f3434d8a031b98f4a496adb8df20c122 (diff) |
UTI: Export results
+ Model version 1.0.0.
- Only some result types are exported.
+ MRR, NDRPDR and SOAK.
- Other result types to be added later.
+ In contrast, all test types are detected.
+ Convert custom classes to JSON-serializable equivalents.
+ Sort dict keys before converting to JSON.
+ Override the order for some known keys.
+ Export sets as sorted arrays.
+ Convert to info content from serialized raw content.
+ Also export outputs for suite setups and teardowns.
+ Info files for setup/teardown exist only temporarily.
+ The data is merged into suite.info.json file.
+ This simplifies presentation of total suite duration.
+ Define model via JSON schema:
- Just test case, suite setup/teardown/suite to be added later.
- Just info, raw to be added later.
+ Proper descriptions.
+ Json is generated from yaml.
+ This is a convenience for maintainers.
+ The officially used schema is the .json one.
+ TODOs written into a separate .txt file.
+ Validate exported instance against the schema.
+ Include format checking.
+ Update CSIT requirements for validation dependencies.
+ This needs python-dateutil==2.8.2, only a patch bump.
+ Compute bandwidth also for soak tests.
+ This unifies with NDRPDR to simplify schema definition.
- PAL may need an update for parsing soak test message.
+ Include SSH log items, raw output only.
+ Generate all outputs in a single filesystem tree.
+ Move raw outputs into test_output_raw.tar.xz.
+ Rename existing tar with suites to generated_robot_files.tar.xz.
Change-Id: I69ff7b330ed1a14dc435fd0ef008e753c0d7f78c
Signed-off-by: Vratko Polak <vrpolak@cisco.com>
Diffstat (limited to 'resources/libraries/python/model/ExportResult.py')
-rw-r--r-- | resources/libraries/python/model/ExportResult.py | 179 |
1 files changed, 179 insertions, 0 deletions
diff --git a/resources/libraries/python/model/ExportResult.py b/resources/libraries/python/model/ExportResult.py new file mode 100644 index 0000000000..d74a6ab5df --- /dev/null +++ b/resources/libraries/python/model/ExportResult.py @@ -0,0 +1,179 @@ +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module with keywords that publish parts of result structure.""" + +from robot.libraries.BuiltIn import BuiltIn + +from resources.libraries.python.model.util import descend, get_export_data + + +def export_dut_type_and_version(dut_type=u"unknown", dut_version=u"unknown"): + """Export the arguments as dut type and version. + + Robot tends to convert "none" into None, hence the unusual default values. + + If either argument is missing, the value from robot variable is used. + If argument is present, the value is also stored to robot suite variable. + + :param dut_type: DUT type, e.g. VPP or DPDK. + :param dut_version: DUT version as determined by the caller. + :type dut_type: Optional[str] + :type dut_version: Optiona[str] + :raises RuntimeError: If value is neither in argument not robot variable. + """ + if dut_type == u"unknown": + dut_type = BuiltIn().get_variable_value(u"\\${DUT_TYPE}", u"unknown") + if dut_type == u"unknown": + raise RuntimeError(u"Dut type not provided.") + else: + # We want to set a variable in higher level suite setup + # to be available to test setup several levels lower. + # Documentation [0] looks like "children" is a keyword argument, + # but code [1] lines 1458 and 1511-1512 show + # it is just last stringy argument. + # [0] http://robotframework.org/robotframework/ + # 3.1.2/libraries/BuiltIn.html#Set%20Suite%20Variable + # [1] https://github.com/robotframework/robotframework/blob/ + # v3.1.2/src/robot/libraries/BuiltIn.py + BuiltIn().set_suite_variable( + u"\\${DUT_TYPE}", dut_type, u"children=True" + ) + if dut_version == u"unknown": + dut_version = BuiltIn().get_variable_value(u"\\${DUT_VERSION}", u"unknown") + if dut_type == u"unknown": + raise RuntimeError(u"Dut version not provided.") + else: + BuiltIn().set_suite_variable( + u"\\${DUT_VERSION}", dut_version, u"children=True" + ) + data = get_export_data() + data[u"dut_type"] = dut_type + data[u"dut_version"] = dut_version + + +def append_mrr_value(mrr_value, unit): + """Store mrr value to proper place so it is dumped into json. + + The value is appended only when unit is not empty. + + :param mrr_value: Forwarding rate from MRR trial. + :param unit: Unit of measurement for the rate. + :type mrr_value: float + :type unit: str + """ + if not unit: + return + data = get_export_data() + data[u"result"][u"type"] = u"mrr" + rate_node = descend(descend(data[u"result"], u"receive_rate"), "rate") + rate_node[u"unit"] = str(unit) + values_list = descend(rate_node, u"values", list) + values_list.append(float(mrr_value)) + # TODO: Fill in the bandwidth part for pps? + + +def export_search_bound(text, value, unit, bandwidth=None): + """Store bound value and unit. + + This function works for both NDRPDR and SOAK, decided by text. + + If a node does not exist, it is created. + If a previous value exists, it is overwritten silently. + Result type is set (overwritten) to ndrpdr (or soak). + + Text is used to determine whether it is ndr or pdr, upper or lower bound, + as the Robot caller has the information only there. + + :param text: Info from Robot caller to determime bound type. + :param value: The bound value in packets (or connections) per second. + :param unit: Rate unit the bound is measured (or estimated) in. + :param bandwidth: The same value recomputed into L1 bits per second. + :type text: str + :type value: float + :type unit: str + :type bandwidth: Optional[float] + """ + value = float(value) + text = str(text).lower() + result_type = u"soak" if u"plrsearch" in text else u"ndrpdr" + upper_or_lower = u"upper" if u"upper" in text else u"lower" + ndr_or_pdr = u"ndr" if u"ndr" in text else u"pdr" + + data = get_export_data() + result_node = data[u"result"] + result_node[u"type"] = result_type + rate_item = dict(rate=dict(value=value, unit=unit)) + if bandwidth: + rate_item[u"bandwidth"] = dict(value=float(bandwidth), unit=u"bps") + if result_type == u"soak": + descend(result_node, u"critical_rate")[upper_or_lower] = rate_item + return + descend(result_node, ndr_or_pdr)[upper_or_lower] = rate_item + + +def _add_latency(result_node, percent, whichward, latency_string): + """Descend to a corresponding node and add values from latency string. + + This is an internal block, moved out from export_ndrpdr_latency, + as it can be called up to 4 times. + + :param result_node: UTI tree node to descend from. + :param percent: Percent value to use in node key (90, 50, 10, 0). + :param whichward: "forward" or "reverse". + :param latency_item: Unidir output from TRex utility, min/avg/max/hdrh. + :type result_node: dict + :type percent: int + :type whichward: str + :latency_string: str + """ + l_min, l_avg, l_max, l_hdrh = latency_string.split(u"/", 3) + whichward_node = descend(result_node, f"latency_{whichward}") + percent_node = descend(whichward_node, f"pdr_{percent}") + percent_node[u"min"] = int(l_min) + percent_node[u"avg"] = int(l_avg) + percent_node[u"max"] = int(l_max) + percent_node[u"hdrh"] = l_hdrh + percent_node[u"unit"] = u"us" + + +def export_ndrpdr_latency(text, latency): + """Store NDRPDR hdrh latency data. + + If "latency" node does not exist, it is created. + If a previous value exists, it is overwritten silently. + + Text is used to determine what percentage of PDR is the load, + as the Robot caller has the information only there. + + Reverse data may be missing, we assume the test was unidirectional. + + :param text: Info from Robot caller to determime load. + :param latency: Output from TRex utility, min/avg/max/hdrh. + :type text: str + :type latency: 1-tuple or 2-tuple of str + """ + data = get_export_data() + result_node = data[u"result"] + percent = 0 + if u"90" in text: + percent = 90 + elif u"50" in text: + percent = 50 + elif u"10" in text: + percent = 10 + _add_latency(result_node, percent, u"forward", latency[0]) + # Else TRex does not support latency measurement for this traffic profile. + if len(latency) < 2: + return + _add_latency(result_node, percent, u"reverse", latency[1]) |