diff options
Diffstat (limited to 'resources')
| -rw-r--r-- | resources/libraries/python/model/validate.py | 2 | ||||
| -rw-r--r-- | resources/model_schema/test_case.schema.yaml | 749 |
2 files changed, 750 insertions, 1 deletions
diff --git a/resources/libraries/python/model/validate.py b/resources/libraries/python/model/validate.py index 47948addeb..ee82d3289f 100644 --- a/resources/libraries/python/model/validate.py +++ b/resources/libraries/python/model/validate.py @@ -31,7 +31,7 @@ def get_validators(): :rtype: Mapping[str, jsonschema.validators.Validator] :raises RuntimeError: If schemas are not readable or not valid. """ - relative_path = "docs/model/current/schema/test_case.info.schema.yaml" + relative_path = "resources/model_schema/test_case.schema.yaml" # Robot is always started when CWD is CSIT_DIR. with open(relative_path, "rt", encoding="utf-8") as file_in: schema = json.loads( diff --git a/resources/model_schema/test_case.schema.yaml b/resources/model_schema/test_case.schema.yaml new file mode 100644 index 0000000000..5d33e0e149 --- /dev/null +++ b/resources/model_schema/test_case.schema.yaml @@ -0,0 +1,749 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.4.0 +$schema: https://json-schema.org/draft/2020-12/schema +description: >- + Schema for output of test case. +allOf: +- description: >- + The main structure, without conditional relations between fields yet. + type: object + additionalProperties: false + properties: + duration: + description: >- + A derived quantity. Difference between start_time and end_time, + in seconds. + $ref: "#/$defs/types/nonnegative_number" + dut_type: + description: >- + DUT type used, e.g. VPP or DPDK. + type: string + minLength: 1 + dut_version: + description: >- + Version string appropriate to DUT type used. + type: string + minLength: 1 + end_time: + description: >- + UTC date and time in RFC 3339 format, specifying calendar time + just before test case ended (at the end of test case teardown). + $ref: "#/$defs/types/date_time" + hosts: + description: >- + Array of hosts this test interacted with. + This can be used for identifying testbed number. + Valid tests shoud interact with at least one DUT or TG. + The array is usually sorted, but that is not a requirement. + type: array + minItems: 1 + items: + description: >- + Host identifier, usually numeric IPv4 address. + type: string + telemetry: + description: >- + Array of telemetry entries. Each entry represent one captured + metric. + type: array + minItems: 0 + items: + description: >- + Telemetry entry. + type: string + message: + description: >- + If passed is true, this value is empty. + Otherwise, value taken directly from TEST_MESSAGE + Robot variable, read at the end of test case + (in test teardown, before export and validation). + It contains information from the exception + that caused the failure, probably with additional + exceptions from teardown keywords. + type: string + passed: + description: >- + Value set accordingly to TEST_STATUS Robot variable, + true if and only if the status is "PASS". + The status is read at the end of test case + (in test teardown, before export and validation). + type: boolean + result: + type: object + allOf: + - description: >- + Sub-schema common for all cases, + only result type identifier defined here. + properties: + type: + description: >- + Identifier of which result type case is applied. + type: string + required: + - type + - oneOf: + - description: >- + Result type for unknown case. + This case represents a test with no specific result + (outside message), e.g. device test; + or a test with result not parsed into + this version of model yet, e.g. GSO test. + additionalProperties: false + properties: + type: + const: unknown + - description: >- + Result type MRR case. + additionalProperties: false + properties: + type: + const: mrr + receive_rate: + description: >- + The results refer to receive rates for multiple + MRR trials. For PPS, these are aggregate + (bidirectional) rates. + Currently, the tests are exporting + approximated receive rates. + That means the actual trial duration + is measured (as opposed to trusting traffic + generator to honor its target duration), + so the resulting values contain noise + from time measurement, and can be lower + than the real performance + (due to various time overheads). + Bandwidth values are supported, but currently + Robot does not export them. + $ref: "#/$defs/types/rate_list_with_bandwidth" + required: + - type + - receive_rate + - description: >- + Result type NDRPDR case. + additionalProperties: false + properties: + type: + const: ndrpdr + ndr: + description: >- + The results refer to search for NDR + (Non Drop Rate). For PPS, this is aggregate + (bidirectional) rate. + Each bound was used as the target load value + in a full-duration trial measurement. + The accepted loss ratio for NDR is exact zero. + Note that packets the Traffic Generator + did not send are also counted as lost packets. + $ref: "#/$defs/macros/lower_and_upper_rate" + pdr: + description: >- + The results refer to search for PDR + (Partial Drop Rate). For PPS, this is aggregate + (bidirectional) rate. + Each bound was used as the target load value + in a full-duration trial measurement. + The accepted loss ratio for PDR is 0.5%. + Note that packets the Traffic Generator + did not send are also counted as lost packets. + $ref: "#/$defs/macros/lower_and_upper_rate" + latency_forward: + description: >- + Object with results related to latency part + of NDRPDR test, for forward traffic direction. + It is the direction used in unidirectional + traffic profiles. + ASTF profiles and IMIX STL profiles + do not support latency information, + so for those tests this object is missing. + It is also missing if Traffic Generator + fails to return valid latency results + for any other reasons, + e.g. latency rate is too high for CPU/NIC used. + $ref: "#/$defs/macros/latency_for_loads" + latency_reverse: + description: >- + Object with results related to latency part + of NDRPDR test, for reverse traffic diration. + This object is not present + when unidirectional traffic profiles are used. + ASTF profiles and IMIX STL profiles + do not support latency information, + so for those tests this object is missing. + It is also missing if Traffic Generator + fails to return valid latency results + for any other reasons, + e.g. latency rate is too high for CPU/NIC used. + $ref: "#/$defs/macros/latency_for_loads" + required: + - type + - ndr + - pdr + - description: >- + Result type SOAK case. + additionalProperties: false + properties: + type: + const: soak + critical_rate: + description: >- + The results refer to bayesian estimate + of critical rate corresponding to + average loss ratio of 10^-7. + For PPS, this is aggregate (bidirectional) rate. + The bounds are computed from + trial measurement results, + but are not equal to any target load used. + Note that packets the Traffic Generator + did not send are also counted as lost packets. + $ref: "#/$defs/macros/lower_and_upper_rate" + required: + - type + - critical_rate + - description: >- + Result type RECONF case. + additionalProperties: false + properties: + type: + const: reconf + aggregate_rate: + description: >- + Load used when reconfiguring, found as NDR lower + bound. This is an aggregate (bidirectional) + rate. Note that packets which the Traffic + Generator did not send, are also counted as lost + packets. + $ref: "#/$defs/types/rate_with_bandwidth" + loss: + description: >- + Number of packets lost during reconfiguration, + with the time that equals packet loss divided by + packet rate. + $ref: "#/$defs/types/packet_with_time" + required: + - type + - loss + - aggregate_rate + - description: >- + Result type HOSTSTACK case. + additionalProperties: false + properties: + type: + const: hoststack + bandwidth: + description: >- + Goodput measured in bits per second. + $ref: "#/$defs/types/bandwidth" + completed_requests: + description: >- + Number of completed requests. + $ref: "#/$defs/types/count_requests" + failed_requests: + description: >- + Number of failed requests. + $ref: "#/$defs/types/count_requests" + retransmits: + description: >- + Number of retransmits. + $ref: "#/$defs/types/count_packets" + latency: + description: >- + Value and unit of latency. + $ref: "#/$defs/types/value_with_unit" + duration: + description: >- + The relative time difference (in seconds) + between program start and end. + $ref: "#/$defs/types/time_quantity" + rate: + description: >- + RPS or CPS rate, with corresponding unit, as + reported by TG. + $ref: "#/$defs/types/rate_without_bandwidth" + required: + - type + - bandwidth + start_time: + description: >- + UTC date and time in RFC 3339 format, specifying calendar time + just after test case started (at the start of test setup). + $ref: "#/$defs/types/date_time" + tags: + description: >- + The list of strings comes directly + from Robot variable TEST_TAGS. + The content should include both static and dynamic tags + at the end of test case (teardown). + type: array + items: + type: string + test_documentation: + description: >- + Value taken directly from TEST_DOCUMENTATION Robot variable. + The content is what you see in suite file + at test case definition, which is usually empty + as CSIT uses data driven test cases. + type: string + test_id: + description: >- + A derived quantity. + It is the most complete and unique identifier for a test case. + This property has a value, of the following form: + {suite_name}.{test_name} + Here, suite name comes from SUITE_NAME robot variable, + test name comes from TEST_NAME robot variable, + but both are converted to lower case, + and spaces are replaced by underscores. + type: string + minLength: 3 + test_name_long: + description: >- + A derived quantity. + This property has a value, of the following form: + {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part} + Here, suite part is very similar to suite tag, + but additionally may contain a prefix describing NIC driver used + (if it is not the default one, drv_vfio_pci for VPP tests). + Any space is replaced by underscore and letters are lower case. + type: string + minLength: 3 + test_name_short: + description: >- + A derived quantity. + This property has a value very similar to suite tag, + but additionally may contain a prefix describing NIC driver used + (if it is not the default one, drv_vfio_pci for VPP tests). + Any space is replaced by underscore and letters are lower case. + type: string + minLength: 3 + test_type: + description: >- + A derived quantity. + Test type identifier, PAL uses it to group similar tests, + e.g. for comparison tables. + Ideally, this information should be parseable from test name, + but the current naming scheme is not simple/consistent enough. + The current implementation queries the robot test tags. + The resulting value is frequently identical to result type, + but this schema version does not require any relation there, + as PAL may want to group tests differently. + type: string + enum: + - device + - gso + - hoststack + - mrr + - ndrpdr + - reconf + - soak + tg_type: + description: >- + TG type used, e.g. TREX. + type: string + minLength: 1 + tg_version: + description: >- + Version string appropriate to TG type used. + type: string + minLength: 1 + version: + description: >- + CSIT model version (semver format) + the exporting code adhered to. + type: string + const: 1.4.0 + required: + - duration + - dut_type + - dut_version + - end_time + - hosts + - telemetry + - message + - passed + - result + - start_time + - tags + - test_documentation + - test_id + - test_name_long + - test_name_short + - test_type + - tg_type + - tg_version + - version +- description: >- + Subschema validating relation between status and message. + oneOf: + - description: >- + Subschema for passing tests, message has to be empty. + type: object + properties: + passed: + const: true + message: + const: "" + - description: >- + Subschema for failing tests, mesage cannot be empty. + type: object + properties: + passed: + const: false + message: + minLength: 1 + +$defs: + types: + nonnegative_number: + type: number + minimum: 0 + positive_number: + type: number + minimum: 1 + nonnegative_integer: + type: integer + minimum: 0 + positive_integer: + type: integer + minimum: 1 + date_time: + type: string + format: date-time + empty_array: + type: array + maxItems: 0 + rate_unit: + description: >- + Packets per second (pps), + connections per second (cps), + requests per second (rps). + type: string + enum: + - pps + - cps + - rps + bandwidth_unit: + description: >- + Unit of measurement for bandwidth values. + Currently a constant, but later versions of model + may allow more units. + enum: + - bps + count_packets: + description: >- + Type, for counting packets. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + A number of packets of interest. + unit: + description: >- + Unit suitable for displaying packet counts. + enum: + - packets + count_requests: + description: >- + Type, for counting requests. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + A number of requests of interest. + unit: + description: >- + Unit suitable for displaying request counts. + enum: + - requests + time_quantity: + description: >- + Reusable type, for various time quantites. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + Unless specified otherwise, this is a duration + between two events. + unit: + description: >- + Only seconds are the unit supported for time + quantities. + enum: + - s + value_with_unit: + description: >- + Reusable composite type, value together with its + unit of measurement. + type: object + additionalProperties: false + properties: + value: + description: >- + Numeric value, context specified elsewhere. + The only assumption is that value is not negative. + $ref: "#/$defs/types/nonnegative_number" + unit: + description: >- + Unit of measurement for the value. + Context and allowed values are specified elsewhere. + type: string + required: + - value + - unit + rate_without_bandwidth: + description: >- + Reusable type, for various rate quantites. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + Unless specified otherwise, + this is the aggregated rate + (sum of both traffic directions). + Depending on the usage, the value can express + intended load, offered load, receive rate, + and various approximations + or estimated bounds thereof. + unit: + description: >- + A transaction rate unit the value is expressed in. + $ref: "#/$defs/types/rate_unit" + bandwidth: + description: >- + Reusable type, for various bandwidth quantites. + allOf: + - $ref: "#/$defs/types/value_with_unit" + - properties: + value: + description: >- + Bandwidth value computed from the corresponding + rate. + unit: + $ref: "#/$defs/types/bandwidth_unit" + rate_with_bandwidth: + description: >- + Reusable composite type, joining primary rate + with optional derived bandwidth. + Not all test types currently compute bandwidth, + even if rate unit is pps. + type: object + additionalProperties: false + properties: + rate: + $ref: "#/$defs/types/rate_without_bandwidth" + bandwidth: + $ref: "#/$defs/types/bandwidth" + required: + - rate + packet_with_time: + description: >- + Reusable composite type, joining packet count with the + time quantity. + type: object + additionalProperties: false + properties: + packet: + $ref: "#/$defs/types/count_packets" + time: + $ref: "#/$defs/types/time_quantity" + required: + - packet + - time + value_list_with_unit_and_stats: + description: >- + Reusable composite type, multiple values together with their + unit of measurement and derived statistics. + type: object + additionalProperties: false + properties: + values: + description: >- + List of values of the same unit, useful for MRR. + type: array + minItmes: 1 + items: + description: >- + Numeric value, context specified elsewhere. The only + assumption is that the value is nonnegative. + $ref: "#/$defs/types/nonnegative_number" + avg: + description: >- + A derived quantity. It is the arithmetic average of the + values list. + $ref: "#/$defs/types/nonnegative_number" + stdev: + description: >- + A derived quantity. It is the standard deviation for the + values list, as computed by jumpavg library. + $ref: "#/$defs/types/nonnegative_number" + unit: + description: >- + Unit of measurement for the values. + Context and allowed values are specified elsewhere. + type: string + required: + - values + - avg + - stdev + - unit + rate_list_without_bandwidth: + description: >- + Reusable composite type, multiple rate values. + allOf: + - $ref: "#/$defs/types/value_list_with_unit_and_stats" + - properties: + values: + items: + description: >- + Unless specified otherwise, + this is the aggregated rate + (sum of both traffic directions). + Depending on the usage, the value can express + intended load, offered load, receive rate, + and various approximations or estimated bounds + thereof. + unit: + $ref: "#/$defs/types/rate_unit" + bandwidth_list: + description: >- + Reusable composite type, multiple bandwidth values. This is a + derived quantity. + allOf: + - $ref: "#/$defs/types/value_list_with_unit_and_stats" + - properties: + values: + items: + description: >- + Unless specified otherwise, + this is the aggregated bandwidth + (sum of both traffic directions). + Depending on the usage, the value can express + intended load, offered load, receive rate, + and various approximations or estimated bounds + thereof. + unit: + $ref: "#/$defs/types/bandwidth_unit" + rate_list_with_bandwidth: + description: >- + Reusable composite type, joining primary rates + with optional derived bandwidths (and stats). + No test types currently computes the bandwidth part. + type: object + additionalProperties: false + properties: + rate: + $ref: "#/$defs/types/rate_list_without_bandwidth" + bandwidth: + $ref: "#/$defs/types/bandwidth_list" + required: + - rate + macros: + lower_and_upper_rate: + type: object + additionalProperties: false + properties: + lower: + description: >- + The lower bound (or min_rate) for the estimate + of a particular searched value. + $ref: "#/$defs/types/rate_with_bandwidth" + upper: + description: >- + The upper bound (or max_rate) for the estimate + of a particular searched value. + $ref: "#/$defs/types/rate_with_bandwidth" + required: + - lower + - upper + latency_numbers: + type: object + additionalProperties: false + properties: + min: + description: >- + Rounded minimal latency time measured in this trial. + See unit property for the unit of measurement. + $ref: "#/$defs/types/nonnegative_integer" + max: + description: >- + Rounded maximal latency time measured in this trial. + See unit property for the unit of measurement. + Zero value is not allowed, as that is one of symptoms + of Traffic Generator failing to get proper latency. + $ref: "#/$defs/types/positive_integer" + avg: + description: >- + Rounded average latency time measured in this trial. + See unit property for the unit of measurement. + $ref: "#/$defs/types/nonnegative_integer" + hdrh: + description: >- + Base64-encoded compressed representation of HDRHistogram + of all latency sample times encountered + in this latency trial. + See unit property for the unit of measurement. + Note that some bins can be several units wide. + type: string + unit: + description: >- + Unit of measurement for latency times. + Currently a constant, but later versions + of the model may allow more values. + type: string + enum: + - us + required: + - avg + - hdrh + - max + - min + - unit + latency_for_loads: + type: object + additionalProperties: false + properties: + pdr_0: + description: >- + Object related to latency measurement performed + at minimal rate (currently 9000 pps per direction). + $ref: "#/$defs/macros/latency_numbers" + pdr_10: + description: >- + Object related to latency measurement performed + at 10% of PDR lower bound, if needed rounded up + to minimal rate (currently 9000 pps per direction). + $ref: "#/$defs/macros/latency_numbers" + pdr_50: + description: >- + Object related to latency measurement performed + at 50% of PDR lower bound, if needed rounded up + to minimal rate (currently 9000 pps per direction). + $ref: "#/$defs/macros/latency_numbers" + pdr_90: + description: >- + Object related to latency measurement performed + at 90% of PDR lower bound, if needed rounded up + to minimal rate (currently 9000 pps per direction). + $ref: "#/$defs/macros/latency_numbers" + required: + - pdr_0 + - pdr_10 + - pdr_50 + - pdr_90 |