feat(model): Move schema to resources

Signed-off-by: pmikus <peter.mikus@protonmail.ch>
Change-Id: I905921cc93436c0e0f1c22dfbdc2af62f4b04ac0

2 files changed, 750 insertions, 1 deletions

diff --git a/resources/libraries/python/model/validate.py b/resources/libraries/python/model/validate.py
index 47948addeb..ee82d3289f 100644
--- a/resources/libraries/python/model/validate.py
+++ b/resources/libraries/python/model/validate.py
@@ -31,7 +31,7 @@ def get_validators():
     :rtype: Mapping[str, jsonschema.validators.Validator]
     :raises RuntimeError: If schemas are not readable or not valid.
     """
-    relative_path = "docs/model/current/schema/test_case.info.schema.yaml"
+    relative_path = "resources/model_schema/test_case.schema.yaml"
     # Robot is always started when CWD is CSIT_DIR.
     with open(relative_path, "rt", encoding="utf-8") as file_in:
         schema = json.loads(
diff --git a/resources/model_schema/test_case.schema.yaml b/resources/model_schema/test_case.schema.yaml
new file mode 100644
index 0000000000..5d33e0e149
--- /dev/null
+++ b/resources/model_schema/test_case.schema.yaml
@@ -0,0 +1,749 @@
+# Copyright (c) 2023 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+$id: https://fd.io/FIXME/CSIT/UTI/test_case/info/1.4.0
+$schema: https://json-schema.org/draft/2020-12/schema
+description: >-
+    Schema for output of test case.
+allOf:
+-   description: >-
+        The main structure, without conditional relations between fields yet.
+    type: object
+    additionalProperties: false
+    properties:
+        duration:
+            description: >-
+                A derived quantity. Difference between start_time and end_time,
+                in seconds.
+            $ref: "#/$defs/types/nonnegative_number"
+        dut_type:
+            description: >-
+                DUT type used, e.g. VPP or DPDK.
+            type: string
+            minLength: 1
+        dut_version:
+            description: >-
+                Version string appropriate to DUT type used.
+            type: string
+            minLength: 1
+        end_time:
+            description: >-
+                UTC date and time in RFC 3339 format, specifying calendar time
+                just before test case ended (at the end of test case teardown).
+            $ref: "#/$defs/types/date_time"
+        hosts:
+            description: >-
+                Array of hosts this test interacted with.
+                This can be used for identifying testbed number.
+                Valid tests shoud interact with at least one DUT or TG.
+                The array is usually sorted, but that is not a requirement.
+            type: array
+            minItems: 1
+            items:
+                description: >-
+                    Host identifier, usually numeric IPv4 address.
+                type: string
+        telemetry:
+            description: >-
+                Array of telemetry entries. Each entry represent one captured
+                metric.
+            type: array
+            minItems: 0
+            items:
+                description: >-
+                    Telemetry entry.
+                type: string
+        message:
+            description: >-
+                If passed is true, this value is empty.
+                Otherwise, value taken directly from TEST_MESSAGE
+                Robot variable, read at the end of test case
+                (in test teardown, before export and validation).
+                It contains information from the exception
+                that caused the failure, probably with additional
+                exceptions from teardown keywords.
+            type: string
+        passed:
+            description: >-
+                Value set accordingly to TEST_STATUS Robot variable,
+                true if and only if the status is "PASS".
+                The status is read at the end of test case
+                (in test teardown, before export and validation).
+            type: boolean
+        result:
+            type: object
+            allOf:
+            -   description: >-
+                    Sub-schema common for all cases,
+                    only result type identifier defined here.
+                properties:
+                    type:
+                        description: >-
+                            Identifier of which result type case is applied.
+                        type: string
+                required:
+                -   type
+            -   oneOf:
+                -   description: >-
+                        Result type for unknown case.
+                        This case represents a test with no specific result
+                        (outside message), e.g. device test;
+                        or a test with result not parsed into
+                        this version of model yet, e.g. GSO test.
+                    additionalProperties: false
+                    properties:
+                        type:
+                            const: unknown
+                -   description: >-
+                        Result type MRR case.
+                    additionalProperties: false
+                    properties:
+                        type:
+                            const: mrr
+                        receive_rate:
+                            description: >-
+                                The results refer to receive rates for multiple
+                                MRR trials. For PPS, these are aggregate
+                                (bidirectional) rates.
+                                Currently, the tests are exporting
+                                approximated receive rates.
+                                That means the actual trial duration
+                                is measured (as opposed to trusting traffic
+                                generator to honor its target duration),
+                                so the resulting values contain noise
+                                from time measurement, and can be lower
+                                than the real performance
+                                (due to various time overheads).
+                                Bandwidth values are supported, but currently
+                                Robot does not export them.
+                            $ref: "#/$defs/types/rate_list_with_bandwidth"
+                    required:
+                    -   type
+                    -   receive_rate
+                -   description: >-
+                        Result type NDRPDR case.
+                    additionalProperties: false
+                    properties:
+                        type:
+                            const: ndrpdr
+                        ndr:
+                            description: >-
+                                The results refer to search for NDR
+                                (Non Drop Rate). For PPS, this is aggregate
+                                (bidirectional) rate.
+                                Each bound was used as the target load value
+                                in a full-duration trial measurement.
+                                The accepted loss ratio for NDR is exact zero.
+                                Note that packets the Traffic Generator
+                                did not send are also counted as lost packets.
+                            $ref: "#/$defs/macros/lower_and_upper_rate"
+                        pdr:
+                            description: >-
+                                The results refer to search for PDR
+                                (Partial Drop Rate). For PPS, this is aggregate
+                                (bidirectional) rate.
+                                Each bound was used as the target load value
+                                in a full-duration trial measurement.
+                                The accepted loss ratio for PDR is 0.5%.
+                                Note that packets the Traffic Generator
+                                did not send are also counted as lost packets.
+                            $ref: "#/$defs/macros/lower_and_upper_rate"
+                        latency_forward:
+                            description: >-
+                                Object with results related to latency part
+                                of NDRPDR test, for forward traffic direction.
+                                It is the direction used in unidirectional
+                                traffic profiles.
+                                ASTF profiles and IMIX STL profiles
+                                do not support latency information,
+                                so for those tests this object is missing.
+                                It is also missing if Traffic Generator
+                                fails to return valid latency results
+                                for any other reasons,
+                                e.g. latency rate is too high for CPU/NIC used.
+                            $ref: "#/$defs/macros/latency_for_loads"
+                        latency_reverse:
+                            description: >-
+                                Object with results related to latency part
+                                of NDRPDR test, for reverse traffic diration.
+                                This object is not present
+                                when unidirectional traffic profiles are used.
+                                ASTF profiles and IMIX STL profiles
+                                do not support latency information,
+                                so for those tests this object is missing.
+                                It is also missing if Traffic Generator
+                                fails to return valid latency results
+                                for any other reasons,
+                                e.g. latency rate is too high for CPU/NIC used.
+                            $ref: "#/$defs/macros/latency_for_loads"
+                    required:
+                    -   type
+                    -   ndr
+                    -   pdr
+                -   description: >-
+                        Result type SOAK case.
+                    additionalProperties: false
+                    properties:
+                        type:
+                            const: soak
+                        critical_rate:
+                            description: >-
+                                The results refer to bayesian estimate
+                                of critical rate corresponding to
+                                average loss ratio of 10^-7.
+                                For PPS, this is aggregate (bidirectional) rate.
+                                The bounds are computed from
+                                trial measurement results,
+                                but are not equal to any target load used.
+                                Note that packets the Traffic Generator
+                                did not send are also counted as lost packets.
+                            $ref: "#/$defs/macros/lower_and_upper_rate"
+                    required:
+                    -   type
+                    -   critical_rate
+                -   description: >-
+                        Result type RECONF case.
+                    additionalProperties: false
+                    properties:
+                        type:
+                            const: reconf
+                        aggregate_rate:
+                            description: >-
+                                Load used when reconfiguring, found as NDR lower
+                                bound. This is an aggregate (bidirectional)
+                                rate. Note that packets which the Traffic
+                                Generator did not send, are also counted as lost
+                                packets.
+                            $ref: "#/$defs/types/rate_with_bandwidth"
+                        loss:
+                            description: >-
+                                Number of packets lost during reconfiguration,
+                                with the time that equals packet loss divided by
+                                packet rate.
+                            $ref: "#/$defs/types/packet_with_time"
+                    required:
+                    -   type
+                    -   loss
+                    -   aggregate_rate
+                -   description: >-
+                        Result type HOSTSTACK case.
+                    additionalProperties: false
+                    properties:
+                        type:
+                            const: hoststack
+                        bandwidth:
+                            description: >-
+                                Goodput measured in bits per second.
+                            $ref: "#/$defs/types/bandwidth"
+                        completed_requests:
+                            description: >-
+                                Number of completed requests.
+                            $ref: "#/$defs/types/count_requests"
+                        failed_requests:
+                            description: >-
+                                Number of failed requests.
+                            $ref: "#/$defs/types/count_requests"
+                        retransmits:
+                            description: >-
+                                Number of retransmits.
+                            $ref: "#/$defs/types/count_packets"
+                        latency:
+                            description: >-
+                                Value and unit of latency.
+                            $ref: "#/$defs/types/value_with_unit"
+                        duration:
+                            description: >-
+                                The relative time difference (in seconds)
+                                between program start and end.
+                            $ref: "#/$defs/types/time_quantity"
+                        rate:
+                            description: >-
+                                RPS or CPS rate, with corresponding unit, as
+                                reported by TG.
+                            $ref: "#/$defs/types/rate_without_bandwidth"
+                    required:
+                    -   type
+                    -   bandwidth
+        start_time:
+            description: >-
+                UTC date and time in RFC 3339 format, specifying calendar time
+                just after test case started (at the start of test setup).
+            $ref: "#/$defs/types/date_time"
+        tags:
+            description: >-
+                The list of strings comes directly
+                from Robot variable TEST_TAGS.
+                The content should include both static and dynamic tags
+                at the end of test case (teardown).
+            type: array
+            items:
+                type: string
+        test_documentation:
+            description: >-
+                Value taken directly from TEST_DOCUMENTATION Robot variable.
+                The content is what you see in suite file
+                at test case definition, which is usually empty
+                as CSIT uses data driven test cases.
+            type: string
+        test_id:
+            description: >-
+                A derived quantity.
+                It is the most complete and unique identifier for a test case.
+                This property has a value, of the following form:
+                {suite_name}.{test_name}
+                Here, suite name comes from SUITE_NAME robot variable,
+                test name comes from TEST_NAME robot variable,
+                but both are converted to lower case,
+                and spaces are replaced by underscores.
+            type: string
+            minLength: 3
+        test_name_long:
+            description: >-
+                A derived quantity.
+                This property has a value, of the following form:
+                {nic_short_name}-{frame_size}-{threads_and_cores}-{suite_part}
+                Here, suite part is very similar to suite tag,
+                but additionally may contain a prefix describing NIC driver used
+                (if it is not the default one, drv_vfio_pci for VPP tests).
+                Any space is replaced by underscore and letters are lower case.
+            type: string
+            minLength: 3
+        test_name_short:
+            description: >-
+                A derived quantity.
+                This property has a value very similar to suite tag,
+                but additionally may contain a prefix describing NIC driver used
+                (if it is not the default one, drv_vfio_pci for VPP tests).
+                Any space is replaced by underscore and letters are lower case.
+            type: string
+            minLength: 3
+        test_type:
+            description: >-
+                A derived quantity.
+                Test type identifier, PAL uses it to group similar tests,
+                e.g. for comparison tables.
+                Ideally, this information should be parseable from test name,
+                but the current naming scheme is not simple/consistent enough.
+                The current implementation queries the robot test tags.
+                The resulting value is frequently identical to result type,
+                but this schema version does not require any relation there,
+                as PAL may want to group tests differently.
+            type: string
+            enum:
+            -   device
+            -   gso
+            -   hoststack
+            -   mrr
+            -   ndrpdr
+            -   reconf
+            -   soak
+        tg_type:
+            description: >-
+                TG type used, e.g. TREX.
+            type: string
+            minLength: 1
+        tg_version:
+            description: >-
+                Version string appropriate to TG type used.
+            type: string
+            minLength: 1
+        version:
+            description: >-
+                CSIT model version (semver format)
+                the exporting code adhered to.
+            type: string
+            const: 1.4.0
+    required:
+    -   duration
+    -   dut_type
+    -   dut_version
+    -   end_time
+    -   hosts
+    -   telemetry
+    -   message
+    -   passed
+    -   result
+    -   start_time
+    -   tags
+    -   test_documentation
+    -   test_id
+    -   test_name_long
+    -   test_name_short
+    -   test_type
+    -   tg_type
+    -   tg_version
+    -   version
+-   description: >-
+        Subschema validating relation between status and message.
+    oneOf:
+    -   description: >-
+            Subschema for passing tests, message has to be empty.
+        type: object
+        properties:
+            passed:
+                const: true
+            message:
+                const: ""
+    -   description: >-
+            Subschema for failing tests, mesage cannot be empty.
+        type: object
+        properties:
+            passed:
+                const: false
+            message:
+                minLength: 1
+
+$defs:
+    types:
+        nonnegative_number:
+            type: number
+            minimum: 0
+        positive_number:
+            type: number
+            minimum: 1
+        nonnegative_integer:
+            type: integer
+            minimum: 0
+        positive_integer:
+            type: integer
+            minimum: 1
+        date_time:
+            type: string
+            format: date-time
+        empty_array:
+            type: array
+            maxItems: 0
+        rate_unit:
+            description: >-
+                Packets per second (pps),
+                connections per second (cps),
+                requests per second (rps).
+            type: string
+            enum:
+            -   pps
+            -   cps
+            -   rps
+        bandwidth_unit:
+            description: >-
+                Unit of measurement for bandwidth values.
+                Currently a constant, but later versions of model
+                may allow more units.
+            enum:
+            -   bps
+        count_packets:
+            description: >-
+                Type, for counting packets.
+            allOf:
+            -   $ref: "#/$defs/types/value_with_unit"
+            -   properties:
+                    value:
+                        description: >-
+                            A number of packets of interest.
+                    unit:
+                        description: >-
+                            Unit suitable for displaying packet counts.
+                        enum:
+                        -   packets
+        count_requests:
+            description: >-
+                Type, for counting requests.
+            allOf:
+            -   $ref: "#/$defs/types/value_with_unit"
+            -   properties:
+                    value:
+                        description: >-
+                            A number of requests of interest.
+                    unit:
+                        description: >-
+                            Unit suitable for displaying request counts.
+                        enum:
+                        -   requests
+        time_quantity:
+            description: >-
+                Reusable type, for various time quantites.
+            allOf:
+            -   $ref: "#/$defs/types/value_with_unit"
+            -   properties:
+                    value:
+                        description: >-
+                            Unless specified otherwise, this is a duration
+                            between two events.
+                    unit:
+                        description: >-
+                            Only seconds are the unit supported for time
+                            quantities.
+                        enum:
+                        -   s
+        value_with_unit:
+            description: >-
+                Reusable composite type, value together with its
+                unit of measurement.
+            type: object
+            additionalProperties: false
+            properties:
+                value:
+                    description: >-
+                        Numeric value, context specified elsewhere.
+                        The only assumption is that value is not negative.
+                    $ref: "#/$defs/types/nonnegative_number"
+                unit:
+                    description: >-
+                        Unit of measurement for the value.
+                        Context and allowed values are specified elsewhere.
+                    type: string
+            required:
+            -   value
+            -   unit
+        rate_without_bandwidth:
+            description: >-
+                Reusable type, for various rate quantites.
+            allOf:
+            -   $ref: "#/$defs/types/value_with_unit"
+            -   properties:
+                    value:
+                        description: >-
+                            Unless specified otherwise,
+                            this is the aggregated rate
+                            (sum of both traffic directions).
+                            Depending on the usage, the value can express
+                            intended load, offered load, receive rate,
+                            and various approximations
+                            or estimated bounds thereof.
+                    unit:
+                        description: >-
+                            A transaction rate unit the value is expressed in.
+                        $ref: "#/$defs/types/rate_unit"
+        bandwidth:
+            description: >-
+                Reusable type, for various bandwidth quantites.
+            allOf:
+            -   $ref: "#/$defs/types/value_with_unit"
+            -   properties:
+                    value:
+                        description: >-
+                            Bandwidth value computed from the corresponding
+                            rate.
+                    unit:
+                        $ref: "#/$defs/types/bandwidth_unit"
+        rate_with_bandwidth:
+            description: >-
+                Reusable composite type, joining primary rate
+                with optional derived bandwidth.
+                Not all test types currently compute bandwidth,
+                even if rate unit is pps.
+            type: object
+            additionalProperties: false
+            properties:
+                rate:
+                    $ref: "#/$defs/types/rate_without_bandwidth"
+                bandwidth:
+                    $ref: "#/$defs/types/bandwidth"
+            required:
+            - rate
+        packet_with_time:
+            description: >-
+                Reusable composite type, joining packet count with the
+                time quantity.
+            type: object
+            additionalProperties: false
+            properties:
+                packet:
+                    $ref: "#/$defs/types/count_packets"
+                time:
+                    $ref: "#/$defs/types/time_quantity"
+            required:
+            - packet
+            - time
+        value_list_with_unit_and_stats:
+            description: >-
+                Reusable composite type, multiple values together with their
+                unit of measurement and derived statistics.
+            type: object
+            additionalProperties: false
+            properties:
+                values:
+                    description: >-
+                        List of values of the same unit, useful for MRR.
+                    type: array
+                    minItmes: 1
+                    items:
+                        description: >-
+                            Numeric value, context specified elsewhere. The only
+                            assumption is that the value is nonnegative.
+                        $ref: "#/$defs/types/nonnegative_number"
+                avg:
+                    description: >-
+                        A derived quantity. It is the arithmetic average of the
+                        values list.
+                    $ref: "#/$defs/types/nonnegative_number"
+                stdev:
+                    description: >-
+                        A derived quantity. It is the standard deviation for the
+                        values list, as computed by jumpavg library.
+                    $ref: "#/$defs/types/nonnegative_number"
+                unit:
+                    description: >-
+                        Unit of measurement for the values.
+                        Context and allowed values are specified elsewhere.
+                    type: string
+            required:
+            -   values
+            -   avg
+            -   stdev
+            -   unit
+        rate_list_without_bandwidth:
+            description: >-
+                Reusable composite type, multiple rate values.
+            allOf:
+            -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
+            -   properties:
+                    values:
+                        items:
+                            description: >-
+                                Unless specified otherwise,
+                                this is the aggregated rate
+                                (sum of both traffic directions).
+                                Depending on the usage, the value can express
+                                intended load, offered load, receive rate,
+                                and various approximations or estimated bounds
+                                thereof.
+                    unit:
+                        $ref: "#/$defs/types/rate_unit"
+        bandwidth_list:
+            description: >-
+                Reusable composite type, multiple bandwidth values. This is a
+                derived quantity.
+            allOf:
+            -   $ref: "#/$defs/types/value_list_with_unit_and_stats"
+            -   properties:
+                    values:
+                        items:
+                            description: >-
+                                Unless specified otherwise,
+                                this is the aggregated bandwidth
+                                (sum of both traffic directions).
+                                Depending on the usage, the value can express
+                                intended load, offered load, receive rate,
+                                and various approximations or estimated bounds
+                                thereof.
+                    unit:
+                        $ref: "#/$defs/types/bandwidth_unit"
+        rate_list_with_bandwidth:
+            description: >-
+                Reusable composite type, joining primary rates
+                with optional derived bandwidths (and stats).
+                No test types currently computes the bandwidth part.
+            type: object
+            additionalProperties: false
+            properties:
+                rate:
+                    $ref: "#/$defs/types/rate_list_without_bandwidth"
+                bandwidth:
+                    $ref: "#/$defs/types/bandwidth_list"
+            required:
+            - rate
+    macros:
+        lower_and_upper_rate:
+            type: object
+            additionalProperties: false
+            properties:
+                lower:
+                    description: >-
+                        The lower bound (or min_rate) for the estimate
+                        of a particular searched value.
+                    $ref: "#/$defs/types/rate_with_bandwidth"
+                upper:
+                    description: >-
+                        The upper bound (or max_rate) for the estimate
+                        of a particular searched value.
+                    $ref: "#/$defs/types/rate_with_bandwidth"
+            required:
+            -   lower
+            -   upper
+        latency_numbers:
+            type: object
+            additionalProperties: false
+            properties:
+                min:
+                    description: >-
+                        Rounded minimal latency time measured in this trial.
+                        See unit property for the unit of measurement.
+                    $ref: "#/$defs/types/nonnegative_integer"
+                max:
+                    description: >-
+                        Rounded maximal latency time measured in this trial.
+                        See unit property for the unit of measurement.
+                        Zero value is not allowed, as that is one of symptoms
+                        of Traffic Generator failing to get proper latency.
+                    $ref: "#/$defs/types/positive_integer"
+                avg:
+                    description: >-
+                        Rounded average latency time measured in this trial.
+                        See unit property for the unit of measurement.
+                    $ref: "#/$defs/types/nonnegative_integer"
+                hdrh:
+                    description: >-
+                        Base64-encoded compressed representation of HDRHistogram
+                        of all latency sample times encountered
+                        in this latency trial.
+                        See unit property for the unit of measurement.
+                        Note that some bins can be several units wide.
+                    type: string
+                unit:
+                    description: >-
+                        Unit of measurement for latency times.
+                        Currently a constant, but later versions
+                        of the model may allow more values.
+                    type: string
+                    enum:
+                    -   us
+            required:
+            -   avg
+            -   hdrh
+            -   max
+            -   min
+            -   unit
+        latency_for_loads:
+            type: object
+            additionalProperties: false
+            properties:
+                pdr_0:
+                    description: >-
+                        Object related to latency measurement performed
+                        at minimal rate (currently 9000 pps per direction).
+                    $ref: "#/$defs/macros/latency_numbers"
+                pdr_10:
+                    description: >-
+                        Object related to latency measurement performed
+                        at 10% of PDR lower bound, if needed rounded up
+                        to minimal rate (currently 9000 pps per direction).
+                    $ref: "#/$defs/macros/latency_numbers"
+                pdr_50:
+                    description: >-
+                        Object related to latency measurement performed
+                        at 50% of PDR lower bound, if needed rounded up
+                        to minimal rate (currently 9000 pps per direction).
+                    $ref: "#/$defs/macros/latency_numbers"
+                pdr_90:
+                    description: >-
+                        Object related to latency measurement performed
+                        at 90% of PDR lower bound, if needed rounded up
+                        to minimal rate (currently 9000 pps per direction).
+                    $ref: "#/$defs/macros/latency_numbers"
+            required:
+            -   pdr_0
+            -   pdr_10
+            -   pdr_50
+            -   pdr_90