From 0968ab87778dd1cad67a49315e2df49b0d076297 Mon Sep 17 00:00:00 2001 From: pmikus Date: Mon, 13 Mar 2023 08:22:17 +0000 Subject: feat(model): Move schema to resources Signed-off-by: pmikus Change-Id: I905921cc93436c0e0f1c22dfbdc2af62f4b04ac0 --- .../current/schema/test_case.info.schema.yaml | 749 --------------------- docs/model/current/top.rst | 75 --- resources/libraries/python/model/validate.py | 2 +- resources/model_schema/test_case.schema.yaml | 749 +++++++++++++++++++++ 4 files changed, 750 insertions(+), 825 deletions(-) delete mode 100644 docs/model/current/schema/test_case.info.schema.yaml delete mode 100644 docs/model/current/top.rst create mode 100644 resources/model_schema/test_case.schema.yaml diff --git a/docs/model/current/schema/test_case.info.schema.yaml b/docs/model/current/schema/test_case.info.schema.yaml deleted file mode 100644 index 5d33e0e149..0000000000 --- a/docs/model/current/schema/test_case.info.schema.yaml +++ /dev/null @@ -1,749 +0,0 @@ -# Copyright (c) 2023 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.4.0 -$schema: https://json-schema.org/draft/2020-12/schema -description: >- - Schema for output of test case. -allOf: -- description: >- - The main structure, without conditional relations between fields yet. - type: object - additionalProperties: false - properties: - duration: - description: >- - A derived quantity. Difference between start_time and end_time, - in seconds. - $ref: "#/$defs/types/nonnegative_number" - dut_type: - description: >- - DUT type used, e.g. VPP or DPDK. - type: string - minLength: 1 - dut_version: - description: >- - Version string appropriate to DUT type used. - type: string - minLength: 1 - end_time: - description: >- - UTC date and time in RFC 3339 format, specifying calendar time - just before test case ended (at the end of test case teardown). - $ref: "#/$defs/types/date_time" - hosts: - description: >- - Array of hosts this test interacted with. - This can be used for identifying testbed number. - Valid tests shoud interact with at least one DUT or TG. - The array is usually sorted, but that is not a requirement. - type: array - minItems: 1 - items: - description: >- - Host identifier, usually numeric IPv4 address. - type: string - telemetry: - description: >- - Array of telemetry entries. Each entry represent one captured - metric. - type: array - minItems: 0 - items: - description: >- - Telemetry entry. - type: string - message: - description: >- - If passed is true, this value is empty. - Otherwise, value taken directly from TEST_MESSAGE - Robot variable, read at the end of test case - (in test teardown, before export and validation). - It contains information from the exception - that caused the failure, probably with additional - exceptions from teardown keywords. - type: string - passed: - description: >- - Value set accordingly to TEST_STATUS Robot variable, - true if and only if the status is "PASS". - The status is read at the end of test case - (in test teardown, before export and validation). - type: boolean - result: - type: object - allOf: - - description: >- - Sub-schema common for all cases, - only result type identifier defined here. - properties: - type: - description: >- - Identifier of which result type case is applied. - type: string - required: - - type - - oneOf: - - description: >- - Result type for unknown case. - This case represents a test with no specific result - (outside message), e.g. device test; - or a test with result not parsed into - this version of model yet, e.g. GSO test. - additionalProperties: false - properties: - type: - const: unknown - - description: >- - Result type MRR case. - additionalProperties: false - properties: - type: - const: mrr - receive_rate: - description: >- - The results refer to receive rates for multiple - MRR trials. For PPS, these are aggregate - (bidirectional) rates. - Currently, the tests are exporting - approximated receive rates. - That means the actual trial duration - is measured (as opposed to trusting traffic - generator to honor its target duration), - so the resulting values contain noise - from time measurement, and can be lower - than the real performance - (due to various time overheads). - Bandwidth values are supported, but currently - Robot does not export them. - $ref: "#/$defs/types/rate_list_with_bandwidth" - required: - - type - - receive_rate - - description: >- - Result type NDRPDR case. - additionalProperties: false - properties: - type: - const: ndrpdr - ndr: - description: >- - The results refer to search for NDR - (Non Drop Rate). For PPS, this is aggregate - (bidirectional) rate. - Each bound was used as the target load value - in a full-duration trial measurement. - The accepted loss ratio for NDR is exact zero. - Note that packets the Traffic Generator - did not send are also counted as lost packets. - $ref: "#/$defs/macros/lower_and_upper_rate" - pdr: - description: >- - The results refer to search for PDR - (Partial Drop Rate). For PPS, this is aggregate - (bidirectional) rate. - Each bound was used as the target load value - in a full-duration trial measurement. - The accepted loss ratio for PDR is 0.5%. - Note that packets the Traffic Generator - did not send are also counted as lost packets. - $ref: "#/$defs/macros/lower_and_upper_rate" - latency_forward: - description: >- - Object with results related to latency part - of NDRPDR test, for forward traffic direction. - It is the direction used in unidirectional - traffic profiles. - ASTF profiles and IMIX STL profiles - do not support latency information, - so for those tests this object is missing. - It is also missing if Traffic Generator - fails to return valid latency results - for any other reasons, - e.g. latency rate is too high for CPU/NIC used. - $ref: "#/$defs/macros/latency_for_loads" - latency_reverse: - description: >- - Object with results related to latency part - of NDRPDR test, for reverse traffic diration. - This object is not present - when unidirectional traffic profiles are used. - ASTF profiles and IMIX STL profiles - do not support latency information, - so for those tests this object is missing. - It is also missing if Traffic Generator - fails to return valid latency results - for any other reasons, - e.g. latency rate is too high for CPU/NIC used. - $ref: "#/$defs/macros/latency_for_loads" - required: - - type - - ndr - - pdr - - description: >- - Result type SOAK case. - additionalProperties: false - properties: - type: - const: soak - critical_rate: - description: >- - The results refer to bayesian estimate - of critical rate corresponding to - average loss ratio of 10^-7. - For PPS, this is aggregate (bidirectional) rate. - The bounds are computed from - trial measurement results, - but are not equal to any target load used. - Note that packets the Traffic Generator - did not send are also counted as lost packets. - $ref: "#/$defs/macros/lower_and_upper_rate" - required: - - type - - critical_rate - - description: >- - Result type RECONF case. - additionalProperties: false - properties: - type: - const: reconf - aggregate_rate: - description: >- - Load used when reconfiguring, found as NDR lower - bound. This is an aggregate (bidirectional) - rate. Note that packets which the Traffic - Generator did not send, are also counted as lost - packets. - $ref: "#/$defs/types/rate_with_bandwidth" - loss: - description: >- - Number of packets lost during reconfiguration, - with the time that equals packet loss divided by - packet rate. - $ref: "#/$defs/types/packet_with_time" - required: - - type - - loss - - aggregate_rate - - description: >- - Result type HOSTSTACK case. - additionalProperties: false - properties: - type: - const: hoststack - bandwidth: - description: >- - Goodput measured in bits per second. - $ref: "#/$defs/types/bandwidth" - completed_requests: - description: >- - Number of completed requests. - $ref: "#/$defs/types/count_requests" - failed_requests: - description: >- - Number of failed requests. - $ref: "#/$defs/types/count_requests" - retransmits: - description: >- - Number of retransmits. - $ref: "#/$defs/types/count_packets" - latency: - description: >- - Value and unit of latency. - $ref: "#/$defs/types/value_with_unit" - duration: - description: >- - The relative time difference (in seconds) - between program start and end. - $ref: "#/$defs/types/time_quantity" - rate: - description: >- - RPS or CPS rate, with corresponding unit, as - reported by TG. - $ref: "#/$defs/types/rate_without_bandwidth" - required: - - type - - bandwidth - start_time: - description: >- - UTC date and time in RFC 3339 format, specifying calendar time - just after test case started (at the start of test setup). - $ref: "#/$defs/types/date_time" - tags: - description: >- - The list of strings comes directly - from Robot variable TEST_TAGS. - The content should include both static and dynamic tags - at the end of test case (teardown). - type: array - items: - type: string - test_documentation: - description: >- - Value taken directly from TEST_DOCUMENTATION Robot variable. - The content is what you see in suite file - at test case definition, which is usually empty - as CSIT uses data driven test cases. - type: string - test_id: - description: >- - A derived quantity. - It is the most complete and unique identifier for a test case. - This property has a value, of the following form: - {suite_name}.{test_name} - Here, suite name comes from SUITE_NAME robot variable, - test name comes from TEST_NAME robot variable, - but both are converted to lower case, - and spaces are replaced by underscores. - type: string - minLength: 3 - test_name_long: - description: >- - A derived quantity. - This property has a value, of the following form: - {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part} - Here, suite part is very similar to suite tag, - but additionally may contain a prefix describing NIC driver used - (if it is not the default one, drv_vfio_pci for VPP tests). - Any space is replaced by underscore and letters are lower case. - type: string - minLength: 3 - test_name_short: - description: >- - A derived quantity. - This property has a value very similar to suite tag, - but additionally may contain a prefix describing NIC driver used - (if it is not the default one, drv_vfio_pci for VPP tests). - Any space is replaced by underscore and letters are lower case. - type: string - minLength: 3 - test_type: - description: >- - A derived quantity. - Test type identifier, PAL uses it to group similar tests, - e.g. for comparison tables. - Ideally, this information should be parseable from test name, - but the current naming scheme is not simple/consistent enough. - The current implementation queries the robot test tags. - The resulting value is frequently identical to result type, - but this schema version does not require any relation there, - as PAL may want to group tests differently. - type: string - enum: - - device - - gso - - hoststack - - mrr - - ndrpdr - - reconf - - soak - tg_type: - description: >- - TG type used, e.g. TREX. - type: string - minLength: 1 - tg_version: - description: >- - Version string appropriate to TG type used. - type: string - minLength: 1 - version: - description: >- - CSIT model version (semver format) - the exporting code adhered to. - type: string - const: 1.4.0 - required: - - duration - - dut_type - - dut_version - - end_time - - hosts - - telemetry - - message - - passed - - result - - start_time - - tags - - test_documentation - - test_id - - test_name_long - - test_name_short - - test_type - - tg_type - - tg_version - - version -- description: >- - Subschema validating relation between status and message. - oneOf: - - description: >- - Subschema for passing tests, message has to be empty. - type: object - properties: - passed: - const: true - message: - const: "" - - description: >- - Subschema for failing tests, mesage cannot be empty. - type: object - properties: - passed: - const: false - message: - minLength: 1 - -$defs: - types: - nonnegative_number: - type: number - minimum: 0 - positive_number: - type: number - minimum: 1 - nonnegative_integer: - type: integer - minimum: 0 - positive_integer: - type: integer - minimum: 1 - date_time: - type: string - format: date-time - empty_array: - type: array - maxItems: 0 - rate_unit: - description: >- - Packets per second (pps), - connections per second (cps), - requests per second (rps). - type: string - enum: - - pps - - cps - - rps - bandwidth_unit: - description: >- - Unit of measurement for bandwidth values. - Currently a constant, but later versions of model - may allow more units. - enum: - - bps - count_packets: - description: >- - Type, for counting packets. - allOf: - - $ref: "#/$defs/types/value_with_unit" - - properties: - value: - description: >- - A number of packets of interest. - unit: - description: >- - Unit suitable for displaying packet counts. - enum: - - packets - count_requests: - description: >- - Type, for counting requests. - allOf: - - $ref: "#/$defs/types/value_with_unit" - - properties: - value: - description: >- - A number of requests of interest. - unit: - description: >- - Unit suitable for displaying request counts. - enum: - - requests - time_quantity: - description: >- - Reusable type, for various time quantites. - allOf: - - $ref: "#/$defs/types/value_with_unit" - - properties: - value: - description: >- - Unless specified otherwise, this is a duration - between two events. - unit: - description: >- - Only seconds are the unit supported for time - quantities. - enum: - - s - value_with_unit: - description: >- - Reusable composite type, value together with its - unit of measurement. - type: object - additionalProperties: false - properties: - value: - description: >- - Numeric value, context specified elsewhere. - The only assumption is that value is not negative. - $ref: "#/$defs/types/nonnegative_number" - unit: - description: >- - Unit of measurement for the value. - Context and allowed values are specified elsewhere. - type: string - required: - - value - - unit - rate_without_bandwidth: - description: >- - Reusable type, for various rate quantites. - allOf: - - $ref: "#/$defs/types/value_with_unit" - - properties: - value: - description: >- - Unless specified otherwise, - this is the aggregated rate - (sum of both traffic directions). - Depending on the usage, the value can express - intended load, offered load, receive rate, - and various approximations - or estimated bounds thereof. - unit: - description: >- - A transaction rate unit the value is expressed in. - $ref: "#/$defs/types/rate_unit" - bandwidth: - description: >- - Reusable type, for various bandwidth quantites. - allOf: - - $ref: "#/$defs/types/value_with_unit" - - properties: - value: - description: >- - Bandwidth value computed from the corresponding - rate. - unit: - $ref: "#/$defs/types/bandwidth_unit" - rate_with_bandwidth: - description: >- - Reusable composite type, joining primary rate - with optional derived bandwidth. - Not all test types currently compute bandwidth, - even if rate unit is pps. - type: object - additionalProperties: false - properties: - rate: - $ref: "#/$defs/types/rate_without_bandwidth" - bandwidth: - $ref: "#/$defs/types/bandwidth" - required: - - rate - packet_with_time: - description: >- - Reusable composite type, joining packet count with the - time quantity. - type: object - additionalProperties: false - properties: - packet: - $ref: "#/$defs/types/count_packets" - time: - $ref: "#/$defs/types/time_quantity" - required: - - packet - - time - value_list_with_unit_and_stats: - description: >- - Reusable composite type, multiple values together with their - unit of measurement and derived statistics. - type: object - additionalProperties: false - properties: - values: - description: >- - List of values of the same unit, useful for MRR. - type: array - minItmes: 1 - items: - description: >- - Numeric value, context specified elsewhere. The only - assumption is that the value is nonnegative. - $ref: "#/$defs/types/nonnegative_number" - avg: - description: >- - A derived quantity. It is the arithmetic average of the - values list. - $ref: "#/$defs/types/nonnegative_number" - stdev: - description: >- - A derived quantity. It is the standard deviation for the - values list, as computed by jumpavg library. - $ref: "#/$defs/types/nonnegative_number" - unit: - description: >- - Unit of measurement for the values. - Context and allowed values are specified elsewhere. - type: string - required: - - values - - avg - - stdev - - unit - rate_list_without_bandwidth: - description: >- - Reusable composite type, multiple rate values. - allOf: - - $ref: "#/$defs/types/value_list_with_unit_and_stats" - - properties: - values: - items: - description: >- - Unless specified otherwise, - this is the aggregated rate - (sum of both traffic directions). - Depending on the usage, the value can express - intended load, offered load, receive rate, - and various approximations or estimated bounds - thereof. - unit: - $ref: "#/$defs/types/rate_unit" - bandwidth_list: - description: >- - Reusable composite type, multiple bandwidth values. This is a - derived quantity. - allOf: - - $ref: "#/$defs/types/value_list_with_unit_and_stats" - - properties: - values: - items: - description: >- - Unless specified otherwise, - this is the aggregated bandwidth - (sum of both traffic directions). - Depending on the usage, the value can express - intended load, offered load, receive rate, - and various approximations or estimated bounds - thereof. - unit: - $ref: "#/$defs/types/bandwidth_unit" - rate_list_with_bandwidth: - description: >- - Reusable composite type, joining primary rates - with optional derived bandwidths (and stats). - No test types currently computes the bandwidth part. - type: object - additionalProperties: false - properties: - rate: - $ref: "#/$defs/types/rate_list_without_bandwidth" - bandwidth: - $ref: "#/$defs/types/bandwidth_list" - required: - - rate - macros: - lower_and_upper_rate: - type: object - additionalProperties: false - properties: - lower: - description: >- - The lower bound (or min_rate) for the estimate - of a particular searched value. - $ref: "#/$defs/types/rate_with_bandwidth" - upper: - description: >- - The upper bound (or max_rate) for the estimate - of a particular searched value. - $ref: "#/$defs/types/rate_with_bandwidth" - required: - - lower - - upper - latency_numbers: - type: object - additionalProperties: false - properties: - min: - description: >- - Rounded minimal latency time measured in this trial. - See unit property for the unit of measurement. - $ref: "#/$defs/types/nonnegative_integer" - max: - description: >- - Rounded maximal latency time measured in this trial. - See unit property for the unit of measurement. - Zero value is not allowed, as that is one of symptoms - of Traffic Generator failing to get proper latency. - $ref: "#/$defs/types/positive_integer" - avg: - description: >- - Rounded average latency time measured in this trial. - See unit property for the unit of measurement. - $ref: "#/$defs/types/nonnegative_integer" - hdrh: - description: >- - Base64-encoded compressed representation of HDRHistogram - of all latency sample times encountered - in this latency trial. - See unit property for the unit of measurement. - Note that some bins can be several units wide. - type: string - unit: - description: >- - Unit of measurement for latency times. - Currently a constant, but later versions - of the model may allow more values. - type: string - enum: - - us - required: - - avg - - hdrh - - max - - min - - unit - latency_for_loads: - type: object - additionalProperties: false - properties: - pdr_0: - description: >- - Object related to latency measurement performed - at minimal rate (currently 9000 pps per direction). - $ref: "#/$defs/macros/latency_numbers" - pdr_10: - description: >- - Object related to latency measurement performed - at 10% of PDR lower bound, if needed rounded up - to minimal rate (currently 9000 pps per direction). - $ref: "#/$defs/macros/latency_numbers" - pdr_50: - description: >- - Object related to latency measurement performed - at 50% of PDR lower bound, if needed rounded up - to minimal rate (currently 9000 pps per direction). - $ref: "#/$defs/macros/latency_numbers" - pdr_90: - description: >- - Object related to latency measurement performed - at 90% of PDR lower bound, if needed rounded up - to minimal rate (currently 9000 pps per direction). - $ref: "#/$defs/macros/latency_numbers" - required: - - pdr_0 - - pdr_10 - - pdr_50 - - pdr_90 diff --git a/docs/model/current/top.rst b/docs/model/current/top.rst deleted file mode 100644 index 721e6b2ff0..0000000000 --- a/docs/model/current/top.rst +++ /dev/null @@ -1,75 +0,0 @@ -.. - Copyright (c) 2023 Cisco and/or its affiliates. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at: -.. - http://www.apache.org/licenses/LICENSE-2.0 -.. - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -CSIT model -^^^^^^^^^^ - -This document describes what is currently implemented in CSIT, -especially the export side (UTI), not import side (PAL). - -Version -~~~~~~~ - -This document is valid for CSIT model version 1.4.0. - -It is recommended to use semantic versioning: https://semver.org/ -That means, if the new model misses a field present in the old model, -bump the major version. If the new model adds a field -not present in the old model, bump the minor version. -Any other edit in the implmenetation (or documentation) bumps the patch version. -If you change value type or formatting, -consider whether the parser (PAL) understands the new value correctly. -Renaming a field is the same as adding a new one and removing the old one. -Parser (PAL) has to know exact major version and minimal minor version, -and unless bugs, it can ignore patch version and bumped minor version. - -UTI -~~~ - -UTI stands for Unified Test Interface. -It mainly focuses on exporting information gathered during test run -into JSON output files. - -Output Structure ------------------ - -UTI outputs come in filesystem tree structure (single tree), where directories -correspond to suite levels and files correspond to suite setup, suite teardown -or any test case at this level of suite. -The directory name comes from SUITE_NAME Robot variable (the last part -as the previous parts are higher level suites), converted to lowercase. -If the suite name contains spaces (Robot converts underscores to spaces), -they are replaced with underscores. - -The filesystem tree is rooted under tests/ (as suites in git are there), -and for each component (test case, suite setup, suite teardown). - -Although we expect only ASCII text in the exported files, -we manipulate files using UTF-8 encoding, -so if Robot Framework uses a non-ascii character, it will be handled. - -JSON schemas ------------- - -CSIT model is formally defined as a collection of JSON schema documents, -one for each output file type. - -The current version specifies only one output file type: -Info output for test case. - -The authoritative JSON schema documents are in JSON format. -Git repository also contains YAML formatted document and conversion utility, -which simplifies maintaining of the JSON document -(no need to track brackets and commas), but are not authoritative. diff --git a/resources/libraries/python/model/validate.py b/resources/libraries/python/model/validate.py index 47948addeb..ee82d3289f 100644 --- a/resources/libraries/python/model/validate.py +++ b/resources/libraries/python/model/validate.py @@ -31,7 +31,7 @@ def get_validators(): :rtype: Mapping[str, jsonschema.validators.Validator] :raises RuntimeError: If schemas are not readable or not valid. """ - relative_path = "docs/model/current/schema/test_case.info.schema.yaml" + relative_path = "resources/model_schema/test_case.schema.yaml" # Robot is always started when CWD is CSIT_DIR. with open(relative_path, "rt", encoding="utf-8") as file_in: schema = json.loads( diff --git a/resources/model_schema/test_case.schema.yaml b/resources/model_schema/test_case.schema.yaml new file mode 100644 index 0000000000..5d33e0e149 --- /dev/null +++ b/resources/model_schema/test_case.schema.yaml @@ -0,0 +1,749 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.4.0 +$schema: https://json-schema.org/draft/2020-12/schema +description: >- + Schema for output of test case. +allOf: +- description: >- + The main structure, without conditional relations between fields yet. + type: object + additionalProperties: false + properties: + duration: + description: >- + A derived quantity. Difference between start_time and end_time, + in seconds. + $ref: "#/$defs/types/nonnegative_number" + dut_type: + description: >- + DUT type used, e.g. VPP or DPDK. + type: string + minLength: 1 + dut_version: + description: >- + Version string appropriate to DUT type used. + type: string + minLength: 1 + end_time: + description: >- + UTC date and time in RFC 3339 format, specifying calendar time + just before test case ended (at the end of test case teardown). + $ref: "#/$defs/types/date_time" + hosts: + description: >- + Array of hosts this test interacted with. + This can be used for identifying testbed number. + Valid tests shoud interact with at least one DUT or TG. + The array is usually sorted, but that is not a requirement. + type: array + minItems: 1 + items: + description: >- + Host identifier, usually numeric IPv4 address. + type: string + telemetry: + description: >- + Array of telemetry entries. Each entry represent one captured + metric. + type: array + minItems: 0 + items: + description: >- + Telemetry entry. + type: string + message: + description: >- + If passed is true, this value is empty. + Otherwise, value taken directly from TEST_MESSAGE + Robot variable, read at the end of test case + (in test teardown, before export and validation). + It contains information from the exception + that caused the failure, probably with additional + exceptions from teardown keywords. + type: string + passed: + description: >- + Value set accordingly to TEST_STATUS Robot variable, + true if and only if the status is "PASS". + The status is read at the end of test case + (in test teardown, before export and validation). + type: boolean + result: + type: object + allOf: + - description: >- + Sub-schema common for all cases, + only result type identifier defined here. + properties: + type: + description: >- + Identifier of which result type case is applied. + type: string + required: + - type + - oneOf: + - description: >- + Result type for unknown case. + This case represents a test with no specific result + (outside message), e.g. device test; + or a test with result not parsed into + this version of model yet, e.g. GSO test. + additionalProperties: false + properties: + type: + const: unknown + - description: >- + Result type MRR case. + additionalProperties: false + properties: + type: + const: mrr + receive_rate: + description: >- + The results refer to receive rates for multiple + MRR trials. For PPS, these are aggregate + (bidirectional) rates. + Currently, the tests are exporting + approximated receive rates. + That means the actual trial duration + is measured (as opposed to trusting traffic + generator to honor its target duration), + so the resulting values contain noise + from time measurement, and can be lower + than the real performance + (due to various time overheads). + Bandwidth values are supported, but currently + Robot does not export them. + $ref: "#/$defs/types/rate_list_with_bandwidth" + required: + - type + - receive_rate + - description: >- + Result type NDRPDR case. + additionalProperties: false + properties: + type: + const: ndrpdr + ndr: + description: >- + The results refer to search for NDR + (Non Drop Rate). For PPS, this is aggregate + (bidirectional) rate. + Each bound was used as the target load value + in a full-duration trial measurement. + The accepted loss ratio for NDR is exact zero. + Note that packets the Traffic Generator + did not send are also counted as lost packets. + $ref: "#/$defs/macros/lower_and_upper_rate" + pdr: + description: >- + The results refer to search for PDR + (Partial Drop Rate). For PPS, this is aggregate + (bidirectional) rate. + Each bound was used as the target load value + in a full-duration trial measurement. + The accepted loss ratio for PDR is 0.5%. + Note that packets the Traffic Generator + did not send are also counted as lost packets. + $ref: "#/$defs/macros/lower_and_upper_rate" + latency_forward: + description: >- + Object with results related to latency part + of NDRPDR test, for forward traffic direction. + It is the direction used in unidirectional + traffic profiles. + ASTF profiles and IMIX STL profiles + do not support latency information, + so for those tests this object is missing. + It is also missing if Traffic Generator + fails to return valid latency results + for any other reasons, + e.g. latency rate is too high for CPU/NIC used. + $ref: "#/$defs/macros/latency_for_loads" + latency_reverse: + description: >- + Object with results related to latency part + of NDRPDR test, for reverse traffic diration. + This object is not present + when unidirectional traffic profiles are used. + ASTF profiles and IMIX STL profiles + do not support latency information, + so for those tests this object is missing. + It is also missing if Traffic Generator + fails to return valid latency results + for any other reasons, + e.g. latency rate is too high for CPU/NIC used. + $ref: "#/$defs/macros/latency_for_loads" + required: + - type + - ndr + - pdr + - description: >- + Result type SOAK case. + additionalProperties: false + properties: + type: + const: soak + critical_rate: + description: >- + The results refer to bayesian estimate + of critical rate corresponding to + average loss ratio of 10^-7. + For PPS, this is aggregate (bidirectional) rate. + The bounds are computed from + trial measurement results, + but are not equal to any target load used. + Note that packets the Traffic Generator + did not send are also counted as lost packets. + $ref: "#/$defs/macros/lower_and_upper_rate" + required: + - type + - critical_rate + - description: >- + Result type RECONF case. + additionalProperties: false + properties: + type: + const: reconf + aggregate_rate: + description: >- + Load used when reconfiguring, found as NDR lower + bound. This is an aggregate (bidirectional) + rate. Note that packets which the Traffic + Generator did not send, are also counted as lost + packets. + $ref: "#/$defs/types/rate_with_bandwidth" + loss: + description: >- + Number of packets lost during reconfiguration, + with the time that equals packet loss divided by + packet rate. + $ref: "#/$defs/types/packet_with_time" + required: + - type + - loss + - aggregate_rate + - description: >- + Result type HOSTSTACK case. + additionalProperties: false + properties: + type: + const: hoststack + bandwidth: + description: >- + Goodput measured in bits per second. + $ref: "#/$defs/types/bandwidth" + completed_requests: + description: >- + Number of completed requests. + $ref: "#/$defs/types/count_requests" + failed_requests: + description: >- + Number of failed requests. + $ref: "#/$defs/types/count_requests" + retransmits: + description: >- + Number of retransmits. + $ref: "#/$defs/types/count_packets" + latency: + description: >- + Value and unit of latency. + $ref: "#/$defs/types/value_with_unit" + duration: + description: >- + The relative time difference (in seconds) + between program start and end. + $ref: "#/$defs/types/time_quantity" + rate: + description: >- + RPS or CPS rate, with corresponding unit, as + reported by TG. + $ref: "#/$defs/types/rate_without_bandwidth" + required: + - type + - bandwidth + start_time: + description: >- + UTC date and time in RFC 3339 format, specifying calendar time + just after test case started (at the start of test setup). + $ref: "#/$defs/types/date_time" + tags: + description: >- + The list of strings comes directly + from Robot variable TEST_TAGS. + The content should include both static and dynamic tags + at the end of test case (teardown). + type: array + items: + type: string + test_documentation: + description: >- + Value taken directly from TEST_DOCUMENTATION Robot variable. + The content is what you see in suite file + at test case definition, which is usually empty + as CSIT uses data driven test cases. + type: string + test_id: + description: >- + A derived quantity. + It is the most complete and unique identifier for a test case. + This property has a value, of the following form: + {suite_name}.{test_name} + Here, suite name comes from SUITE_NAME robot variable, + test name comes from TEST_NAME robot variable, + but both are converted to lower case, + and spaces are replaced by underscores. + type: string + minLength: 3 + test_name_long: + description: >- + A derived quantity. + This property has a value, of the following form: + {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part} + Here, suite part is very similar to suite tag, + but additionally may contain a prefix describing NIC driver used + (if it is not the default one, drv_vfio_pci for VPP tests). + Any space is replaced by underscore and letters are lower case. + type: string + minLength: 3 + test_name_short: + description: >- + A derived quantity. + This property has a value very similar to suite tag, + but additionally may contain a prefix describing NIC driver used + (if it is not the default one, drv_vfio_pci for VPP tests). + Any space is replaced by underscore and letters are lower case. + type: string + minLength: 3 + test_type: + description: >- + A derived quantity. + Test type identifier, PAL uses it to group similar tests, + e.g. for comparison tables. + Ideally, this information should be parseable from test name, + but the current naming scheme is not simple/consistent enough. + The current implementation queries the robot test tags. + The resulting value is frequently identical to result type, + but this schema version does not require any relation there, + as PAL may want to group tests differently. + type: string + enum: + - device + - gso + - hoststack + - mrr + - ndrpdr + - reconf + - soak + tg_type: + description: >- + TG type used, e.g. TREX. + type: string + minLength: 1 + tg_version: + description: >- + Version string appropriate to TG type used. + type: string + minLength: 1 + version: + description: >- + CSIT model version (semver format) + the exporting code adhered to. + type: string + const: 1.4.0 + required: + - duration + - dut_type + - dut_version + - end_time + - hosts + - telemetry + - message + - passed + - result + - start_time + - tags + - test_documentation + - test_id + - test_name_long + - test_name_short + - test_type + - tg_type + - tg_version + - version +- description: >- + Subschema validating relation between status and message. + oneOf: + - description: >- + Subschema for passing tests, message has to be empty. + type: object + properties: + passed: + const: true + message: + const: "" + - description: >- + Subschema for failing tests, mesage cannot be empty. + type: object + properties: + passed: + const: false + message: + minLength: 1 + +$defs: + types: + nonnegative_number: + type: number + minimum: 0 + positive_number: + type: number + minimum: 1 + nonnegative_integer: + type: integer + minimum: 0 + positive_integer: + type: integer + minimum: 1 + date_time: + type: string + format: date-time + empty_array: + type: array + maxItems: 0 + rate_unit: + description: >- + Packets per second (pps), + connections per second (cps), + requests per second (rps). + type: string + enum: + - pps + - cps + - rps + bandwidth_unit: + description: >- + Unit of measurement for bandwidth values. + Currently a constant, but later versions of model + may allow more units. + enum: + - bps + count_packets: + description: >- + Type, for counting packets. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + A number of packets of interest. + unit: + description: >- + Unit suitable for displaying packet counts. + enum: + - packets + count_requests: + description: >- + Type, for counting requests. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + A number of requests of interest. + unit: + description: >- + Unit suitable for displaying request counts. + enum: + - requests + time_quantity: + description: >- + Reusable type, for various time quantites. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + Unless specified otherwise, this is a duration + between two events. + unit: + description: >- + Only seconds are the unit supported for time + quantities. + enum: + - s + value_with_unit: + description: >- + Reusable composite type, value together with its + unit of measurement. + type: object + additionalProperties: false + properties: + value: + description: >- + Numeric value, context specified elsewhere. + The only assumption is that value is not negative. + $ref: "#/$defs/types/nonnegative_number" + unit: + description: >- + Unit of measurement for the value. + Context and allowed values are specified elsewhere. + type: string + required: + - value + - unit + rate_without_bandwidth: + description: >- + Reusable type, for various rate quantites. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + Unless specified otherwise, + this is the aggregated rate + (sum of both traffic directions). + Depending on the usage, the value can express + intended load, offered load, receive rate, + and various approximations + or estimated bounds thereof. + unit: + description: >- + A transaction rate unit the value is expressed in. + $ref: "#/$defs/types/rate_unit" + bandwidth: + description: >- + Reusable type, for various bandwidth quantites. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + Bandwidth value computed from the corresponding + rate. + unit: + $ref: "#/$defs/types/bandwidth_unit" + rate_with_bandwidth: + description: >- + Reusable composite type, joining primary rate + with optional derived bandwidth. + Not all test types currently compute bandwidth, + even if rate unit is pps. + type: object + additionalProperties: false + properties: + rate: + $ref: "#/$defs/types/rate_without_bandwidth" + bandwidth: + $ref: "#/$defs/types/bandwidth" + required: + - rate + packet_with_time: + description: >- + Reusable composite type, joining packet count with the + time quantity. + type: object + additionalProperties: false + properties: + packet: + $ref: "#/$defs/types/count_packets" + time: + $ref: "#/$defs/types/time_quantity" + required: + - packet + - time + value_list_with_unit_and_stats: + description: >- + Reusable composite type, multiple values together with their + unit of measurement and derived statistics. + type: object + additionalProperties: false + properties: + values: + description: >- + List of values of the same unit, useful for MRR. + type: array + minItmes: 1 + items: + description: >- + Numeric value, context specified elsewhere. The only + assumption is that the value is nonnegative. + $ref: "#/$defs/types/nonnegative_number" + avg: + description: >- + A derived quantity. It is the arithmetic average of the + values list. + $ref: "#/$defs/types/nonnegative_number" + stdev: + description: >- + A derived quantity. It is the standard deviation for the + values list, as computed by jumpavg library. + $ref: "#/$defs/types/nonnegative_number" + unit: + description: >- + Unit of measurement for the values. + Context and allowed values are specified elsewhere. + type: string + required: + - values + - avg + - stdev + - unit + rate_list_without_bandwidth: + description: >- + Reusable composite type, multiple rate values. + allOf: + - $ref: "#/$defs/types/value_list_with_unit_and_stats" + - properties: + values: + items: + description: >- + Unless specified otherwise, + this is the aggregated rate + (sum of both traffic directions). + Depending on the usage, the value can express + intended load, offered load, receive rate, + and various approximations or estimated bounds + thereof. + unit: + $ref: "#/$defs/types/rate_unit" + bandwidth_list: + description: >- + Reusable composite type, multiple bandwidth values. This is a + derived quantity. + allOf: + - $ref: "#/$defs/types/value_list_with_unit_and_stats" + - properties: + values: + items: + description: >- + Unless specified otherwise, + this is the aggregated bandwidth + (sum of both traffic directions). + Depending on the usage, the value can express + intended load, offered load, receive rate, + and various approximations or estimated bounds + thereof. + unit: + $ref: "#/$defs/types/bandwidth_unit" + rate_list_with_bandwidth: + description: >- + Reusable composite type, joining primary rates + with optional derived bandwidths (and stats). + No test types currently computes the bandwidth part. + type: object + additionalProperties: false + properties: + rate: + $ref: "#/$defs/types/rate_list_without_bandwidth" + bandwidth: + $ref: "#/$defs/types/bandwidth_list" + required: + - rate + macros: + lower_and_upper_rate: + type: object + additionalProperties: false + properties: + lower: + description: >- + The lower bound (or min_rate) for the estimate + of a particular searched value. + $ref: "#/$defs/types/rate_with_bandwidth" + upper: + description: >- + The upper bound (or max_rate) for the estimate + of a particular searched value. + $ref: "#/$defs/types/rate_with_bandwidth" + required: + - lower + - upper + latency_numbers: + type: object + additionalProperties: false + properties: + min: + description: >- + Rounded minimal latency time measured in this trial. + See unit property for the unit of measurement. + $ref: "#/$defs/types/nonnegative_integer" + max: + description: >- + Rounded maximal latency time measured in this trial. + See unit property for the unit of measurement. + Zero value is not allowed, as that is one of symptoms + of Traffic Generator failing to get proper latency. + $ref: "#/$defs/types/positive_integer" + avg: + description: >- + Rounded average latency time measured in this trial. + See unit property for the unit of measurement. + $ref: "#/$defs/types/nonnegative_integer" + hdrh: + description: >- + Base64-encoded compressed representation of HDRHistogram + of all latency sample times encountered + in this latency trial. + See unit property for the unit of measurement. + Note that some bins can be several units wide. + type: string + unit: + description: >- + Unit of measurement for latency times. + Currently a constant, but later versions + of the model may allow more values. + type: string + enum: + - us + required: + - avg + - hdrh + - max + - min + - unit + latency_for_loads: + type: object + additionalProperties: false + properties: + pdr_0: + description: >- + Object related to latency measurement performed + at minimal rate (currently 9000 pps per direction). + $ref: "#/$defs/macros/latency_numbers" + pdr_10: + description: >- + Object related to latency measurement performed + at 10% of PDR lower bound, if needed rounded up + to minimal rate (currently 9000 pps per direction). + $ref: "#/$defs/macros/latency_numbers" + pdr_50: + description: >- + Object related to latency measurement performed + at 50% of PDR lower bound, if needed rounded up + to minimal rate (currently 9000 pps per direction). + $ref: "#/$defs/macros/latency_numbers" + pdr_90: + description: >- + Object related to latency measurement performed + at 90% of PDR lower bound, if needed rounded up + to minimal rate (currently 9000 pps per direction). + $ref: "#/$defs/macros/latency_numbers" + required: + - pdr_0 + - pdr_10 + - pdr_50 + - pdr_90 -- cgit 1.2.3-korg