CSIT-1186: Add srv6 NDRPDR suites

Delete the corresponding ndrpdrdisc suites.

MRR suites are also edited,
to make them less different from NDRPDR suites.

Now, there is a script to re-generate testcases of selected suites.
For that, resources/libraries/python/autogen is created,
with Regenerator holding the most of script logic.

The script looks at files matching a glob expression,
reuses a "prolog", but overwites test cases with the default template
and hardcoded set of testcase arguments.

Also, handling of overhead size has been simplified,
using new keyword Get Max Rate And Jumbo And Handle Multi Seg.

Default value for search timeout is increased to 12 minutes
to make timeout failures less frequent.

Change-Id: I394ad61f262b3078e5ca719034b15ada48987ee0
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

7 files changed, 360 insertions, 19 deletions

diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index 9959f25ae7..8acf1eb28b 100644
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -613,7 +613,7 @@ class OptimizedSearch(object):
             maximum_transmit_rate, packet_loss_ratio=0.005,
             final_relative_width=0.005, final_trial_duration=30.0,
             initial_trial_duration=1.0, number_of_intermediate_phases=2,
-            timeout=600.0):
+            timeout=720.0):
         """Setup initialized TG, perform optimized search, return intervals.
 
         :param frame_size: Frame size identifier or value [B].
diff --git a/resources/libraries/python/autogen/Copyright.py b/resources/libraries/python/autogen/Copyright.py
new file mode 100644
index 0000000000..e8c72f0624
--- /dev/null
+++ b/resources/libraries/python/autogen/Copyright.py
@@ -0,0 +1,29 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module defining sole constant holding copyright text for current year."""
+
+
+COPYRIGHT = '''# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
\ No newline at end of file
diff --git a/resources/libraries/python/autogen/DefaultTestcase.py b/resources/libraries/python/autogen/DefaultTestcase.py
new file mode 100644
index 0000000000..9878289bd1
--- /dev/null
+++ b/resources/libraries/python/autogen/DefaultTestcase.py
@@ -0,0 +1,34 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module with utilities for autogeneration of non-customizable testcases."""
+
+from Testcase import Testcase
+
+
+class DefaultTestcase(Testcase):
+    """Testcase subclass with a rigid template string."""
+
+    def __init__(self, suite_id):
+        """Construct instance for identified suite.
+
+        :param suite_id: Suite identifier, without NIC prefix and .robot suffix.
+            Example: ethip6srhip6-ip6base-srv6enc2sids-nodecaps-ndrpdr
+        :type suite_id: str
+        """
+        template_string = r'''
+| ${tc_num}-${frame_str}-${cores_str}c-''' + suite_id + r'''
+| | [Tags] | ${frame_str} | ${cores_str}C
+| | framesize=${frame_num} | phy_cores=${cores_num}
+'''
+        super(DefaultTestcase, self).__init__(template_string)
diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py
new file mode 100644
index 0000000000..85e8b60dd5
--- /dev/null
+++ b/resources/libraries/python/autogen/Regenerator.py
@@ -0,0 +1,96 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module defining utilities for test directory regeneration."""
+
+from glob import glob
+from os import getcwd
+
+from .Testcase import Testcase
+from .DefaultTestcase import DefaultTestcase
+
+
+class Regenerator(object):
+    """Class containing file generating methods."""
+
+    def __init__(self, testcase_class=DefaultTestcase):
+        """Initialize Testcase class to use.
+
+        TODO: See the type doc for testcase_class?
+        It implies the design is wrong. Fix it.
+        Easiest: Hardcode Regenerator to use DefaultTestcase only.
+
+        :param testcase_class: Subclass of DefaultTestcase for generation.
+            Default: DefaultTestcase
+        :type testcase_class: subclass of DefaultTestcase accepting suite_id
+        """
+        self.testcase_class = testcase_class
+
+    def regenerate_glob(self, pattern, is_ip6=False, tc_kwargs_list=None):
+        """Regenerate files matching glob pattern based on arguments.
+
+        In the current working directory, find all files matching
+        the glob pattern. Use testcase template (from init) to regenerate
+        test cases, autonumbering them, taking arguments from list.
+        If the list is None, use default list, which depends on ip6 usage.
+
+        :param pattern: Glob pattern to select files. Example: *-ndrpdr.robot
+        :param is_ip6: Flag determining minimal frame size. Default: False
+        :param tc_kwargs_list: Arguments defining the testcases. Default: None
+            When None, default list is used.
+            List item is a dict, argument names are keys.
+            The initialized testcase_class should accept those, and "num".
+            DefaultTestcase accepts "framesize" and "phy_cores".
+        :type pattern: str
+        :type is_ip6: boolean
+        :type tc_kwargs_list: list of tuple or None
+        """
+
+        def add_testcase(file_out, num, **kwargs):
+            file_out.write(testcase.generate(num=num, **kwargs))
+            return num + 1
+
+        def add_testcases(file_out, tc_kwargs_list):
+            num = 1
+            for tc_kwargs in tc_kwargs_list:
+                num = add_testcase(file_out, num, **tc_kwargs)
+
+        print "Regenerator starts at {cwd}".format(cwd=getcwd())
+        min_framesize = 78 if is_ip6 else 64
+        kwargs_list = tc_kwargs_list if tc_kwargs_list is not None else [
+            {"framesize": min_framesize, "phy_cores": 1},
+            {"framesize": min_framesize, "phy_cores": 2},
+            {"framesize": min_framesize, "phy_cores": 4},
+            {"framesize": 1518, "phy_cores": 1},
+            {"framesize": 1518, "phy_cores": 2},
+            {"framesize": 1518, "phy_cores": 4},
+            {"framesize": 9000, "phy_cores": 1},
+            {"framesize": 9000, "phy_cores": 2},
+            {"framesize": 9000, "phy_cores": 4},
+            {"framesize": "IMIX_v4_1", "phy_cores": 1},
+            {"framesize": "IMIX_v4_1", "phy_cores": 2},
+            {"framesize": "IMIX_v4_1", "phy_cores": 4}
+        ]
+        for filename in glob(pattern):
+            with open(filename, "r") as file_in:
+                text = file_in.read()
+                text_prolog = "".join(text.partition("*** Test Cases ***")[:-1])
+            # TODO: Make the following work for 2n suites.
+            suite_id = filename.split("-", 1)[1].split(".", 1)[0]
+            print "Regenerating suite_id:", suite_id
+            testcase = self.testcase_class(suite_id)
+            with open(filename, "w") as file_out:
+                file_out.write(text_prolog)
+                add_testcases(file_out, kwargs_list)
+        print "Regenerator ends."
+        print  # To make autogen check output more readable.
diff --git a/resources/libraries/python/autogen/Testcase.py b/resources/libraries/python/autogen/Testcase.py
new file mode 100644
index 0000000000..4f92e6c2e1
--- /dev/null
+++ b/resources/libraries/python/autogen/Testcase.py
@@ -0,0 +1,70 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module defining utilities for testcase autogeneration."""
+
+from string import Template
+
+
+class Testcase(object):
+    """Class containing a template string and a substitution method."""
+
+    def __init__(self, template_string):
+        """Construct instance by storing template given by string.
+
+        :param template_string: Template string to generate test case code with.
+            See string.Template documentation for template string syntax.
+            Only the following placeholders are supported:
+            - cores_num - Number of cores as robot number, example: "${2}".
+            - cores_str - Number of physical cores to use, example: "2".
+            - frame_num - Framesize as a number, example: "${74}".
+            - frame_str - Framesize in upper case, example: "74B".
+            - tc_num - Start of testcase name, example: "tc04".
+        :type template_string: str
+        """
+        self.template = Template(template_string)
+
+    def generate(self, num, framesize, phy_cores):
+        """Return string of test case code with placeholders filled.
+
+        Fail if there are placeholders left unfilled.
+
+        :param num: Test case number. Example value: 4.
+        :param framesize: Imix string or numeric frame size. Example: 74.
+        :param phy_cores: Number of physical cores to use. Example: 2.
+        :type num: int
+        :type framesize: str or int
+        :type phy_cores: int or str
+        :returns: Filled template, usable as test case code.
+        :rtype: str
+        """
+        try:
+            fs = int(framesize)
+            subst_dict = {
+                "frame_num": "${%d}" % fs,
+                "frame_str": "%dB" % fs
+            }
+        except ValueError:  # Assuming an IMIX string.
+            subst_dict = {
+                "frame_num": str(framesize),
+                "frame_str": "IMIX"
+            }
+        cores_str = str(phy_cores)
+        cores_num = int(cores_str)
+        subst_dict.update(
+            {
+                "cores_num": "${%d}" % cores_num,
+                "cores_str": phy_cores,
+                "tc_num": "tc{num:02d}".format(num=num)
+            })
+        return self.template.substitute(subst_dict)
diff --git a/resources/libraries/python/autogen/__init__.py b/resources/libraries/python/autogen/__init__.py
new file mode 100644
index 0000000000..dfb2255a09
--- /dev/null
+++ b/resources/libraries/python/autogen/__init__.py
@@ -0,0 +1,16 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+__init__ file for directory resources/libraries/python/autogen
+"""
diff --git a/resources/libraries/robot/performance/performance_utils.robot b/resources/libraries/robot/performance/performance_utils.robot
index 2cb8d07cb4..4cabe5660e 100644
--- a/resources/libraries/robot/performance/performance_utils.robot
+++ b/resources/libraries/robot/performance/performance_utils.robot
@@ -38,11 +38,99 @@
 | ... | and PDR.
 
 *** Keywords ***
+| Get Max Rate And Jumbo
+| | [Documentation]
+| | ... | Argument framesize can be either integer in case of a single packet
+| | ... | in stream, or IMIX string defining mix of packets.
+| | ... | For jumbo frames detection, the maximal packet size is relevant.
+| | ... | For maximal transmit rate, the average packet size is relevant.
+| | ... | In both cases, encapsulation overhead (if any) has effect.
+| | ... | The maximal rate is computed from line limit bandwidth,
+| | ... | but NICs also have an independent limit in packet rate.
+| | ... | For some NICs, the limit is not reachable (bps limit is stricter),
+| | ... | in those cases None is used (meaning no limiting).
+| | ...
+| | ... | This keyword returns computed maximal unidirectional transmit rate
+| | ... | and jumbo boolean (some suites need that).
+| | ...
+| | ... | *Arguments:*
+| | ... | - bps_limit - Line rate limit in bps. Type: integer
+| | ... | - framesize - Framesize in bytes or IMIX. Type: integer or string
+| | ... | - overhead - Overhead in bytes. Default: 0. Type: integer
+| | ... | - pps_limit - NIC limit rate value in pps. Type: integer or None
+| | ...
+| | ... | *Returns:*
+| | ... | - 2-tuple, consisting of:
+| | ... |   - Calculated unidirectional maximal transmit rate.
+| | ... |     Type: integer or float
+| | ... |   - Jumbo boolean, true if jumbo packet support has to be enabled.
+| | ... |     Type: boolean
+| | ...
+| | ... | *Example:*
+| | ...
+| | ... | \| Get Max Rate And Jumbo | \${10000000} \| IMIX_v4_1 \
+| | ... | \| overhead=\${40} \| pps_limit=\${18750000} \|
+| | ...
+| | [Arguments] | ${bps_limit} | ${framesize}
+| | ... | ${overhead}=${0} | ${pps_limit}=${None}
+| | ...
+| | ${avg_size} = | Set Variable If | '${framesize}' == 'IMIX_v4_1'
+| | ... | ${353.83333} | ${framesize}
+| | ${max_size} = | Set Variable If | '${framesize}' == 'IMIX_v4_1'
+| | ... | ${1518} | ${framesize}
+| | # swo := size_with_overhead
+| | ${avg_swo} = | Evaluate | ${avg_size} + ${overhead}
+| | ${max_swo} = | Evaluate | ${max_size} + ${overhead}
+| | ${jumbo} = | Set Variable If | ${max_swo} < 1522
+| | ... | ${False} | ${True}
+| | # For testing None see: https://groups.google.com/\
+| | #                       forum/#!topic/robotframework-users/XntFz0ocD9E
+| | ${limit_set} = | Set Variable | ${pps_limit != None}
+| | ${rate} = | Evaluate | (${bps_limit}/((${avg_swo}+20)*8)).__trunc__()
+| | ${max_rate} = | Set Variable If | ${limit_set} and ${rate} > ${pps_limit}
+| | ... | ${pps_limit} | ${rate}
+| | Return From Keyword | ${max_rate} | ${jumbo}
+
+| Get Max Rate And Jumbo And Handle Multi Seg
+| | [Documentation]
+| | ... | This keyword adds correct multi seg configuration,
+| | ... | then returns the result of Get Max Rate And Jumbo keyword.
+| | ...
+| | ... | See Documentation of Get Max Rate And Jumbo for more details.
+| | ...
+| | ... | *Arguments:*
+| | ... | - bps_limit - Line rate limit in bps. Type: integer
+| | ... | - framesize - Framesize in bytes. Type: integer or string
+| | ... | - overhead - Overhead in bytes. Default: 0. Type: integer
+| | ... | - pps_limit - NIC limit rate value in pps. Type: integer or None
+| | ...
+| | ... | *Returns:*
+| | ... | - 2-tuple, consisting of:
+| | ... |   - Calculated unidirectional maximal transmit rate.
+| | ... |     Type: integer or float
+| | ... |   - Jumbo boolean, true if jumbo packet support has to be enabled.
+| | ... |     Type: boolean
+| | ...
+| | ... | *Example:*
+| | ...
+| | ... | \| Get Max Rate And Jumbo And Handle Multi Seg | \${10000000} \
+| | ... | \| IMIX_v4_1 \| overhead=\${40} \| pps_limit=\${18750000} \|
+| | ...
+| | [Arguments] | ${bps_limit} | ${framesize}
+| | ... | ${overhead}=${0} | ${pps_limit}=${None}
+| | ...
+| | ${max_rate} | ${jumbo} = | Get Max Rate And Jumbo
+| | ... | ${bps_limit} | ${framesize} | ${overhead} | ${pps_limit}
+| | Run Keyword If | not ${jumbo} | Add no multi seg to all DUTs
+| | Return From Keyword | ${max_rate} | ${jumbo}
+
 | Calculate pps
 | | [Documentation]
 | | ... | Calculate pps for given rate and L2 frame size,
 | | ... | additional 20B are added to L2 frame size as padding.
 | | ...
+| | ... | FIXME: Migrate callers to Get Max Rate And Jumbo
+| | ...
 | | ... | *Arguments*
 | | ... | - bps - Rate in bps. Type: integer
 | | ... | - framesize - L2 frame size in Bytes. Type: integer
@@ -66,6 +154,8 @@
 | | ... | in stream, or set of packets in case of IMIX type or simmilar.
 | | ... | This keyword returns average framesize.
 | | ...
+| | ... | FIXME: Migrate callers to Get Max Rate And Jumbo
+| | ...
 | | ... | *Arguments:*
 | | ... | - framesize - Framesize. Type: integer or string
 | | ...
@@ -362,6 +452,9 @@
 | | ... | Display results as formatted test message.
 | | ... | Fail if a resulting lower bound has too high loss fraction.
 | | ... | Proceed with Perform additional measurements based on NDRPDR result.
+| | ... | Input rates are understood as uni-directional,
+| | ... | reported result contains bi-directional rates.
+| | ...
 | | ... | TODO: Should the trial duration of the additional
 | | ... | measurements be configurable?
 | | ...
@@ -381,19 +474,20 @@
 | | ...
 | | ... | \| Find NDR and PDR intervals using optimized search \| \${64} \| \
 | | ... | 3-node-IPv4 \| \${100000} \| \${14880952} \| \${0.005} \| \${0.005} \
-| | ... | \| \${30.0} \| \${1.0} \| \${2} \| ${600.0}
+| | ... | \| \${30.0} \| \${1.0} \| \${2} \| ${600.0} \|
 | | ...
 | | [Arguments] | ${frame_size} | ${topology_type} | ${minimum_transmit_rate}
 | | ... | ${maximum_transmit_rate} | ${packet_loss_ratio}=${0.005}
 | | ... | ${final_relative_width}=${0.005} | ${final_trial_duration}=${30.0}
 | | ... | ${initial_trial_duration}=${1.0}
-| | ... | ${number_of_intermediate_phases}=${2} | ${timeout}=${600.0}
-| | ...
-| | ${result}= | Perform optimized ndrpdr search | ${frame_size}
-| | ... | ${topology_type} | ${minimum_transmit_rate} | ${maximum_transmit_rate}
-| | ... | ${packet_loss_ratio} | ${final_relative_width}
-| | ... | ${final_trial_duration} | ${initial_trial_duration}
-| | ... | ${number_of_intermediate_phases} | timeout=${timeout}
+| | ... | ${number_of_intermediate_phases}=${2} | ${timeout}=${720.0}
+| | ...
+| | ${result} = | Perform optimized ndrpdr search | ${frame_size}
+| | ... | ${topology_type} | ${minimum_transmit_rate*2}
+| | ... | ${maximum_transmit_rate*2} | ${packet_loss_ratio}
+| | ... | ${final_relative_width} | ${final_trial_duration}
+| | ... | ${initial_trial_duration} | ${number_of_intermediate_phases}
+| | ... | timeout=${timeout}
 | | Display result of NDRPDR search | ${result} | ${frame_size}
 | | Check NDRPDR interval validity | ${result.pdr_interval}
 | | ... | ${packet_loss_ratio}
@@ -421,7 +515,7 @@
 | | ... | *Example:*
 | | ...
 | | ... | \| Display single bound \| NDR lower bound \| \${12345.67} \
-| | ... | \| \${64} \| show_latency=\${EMPTY}
+| | ... | \| \${64} \| show_latency=\${EMPTY} \|
 | | ...
 | | [Arguments] | ${text} | ${rate_total} | ${framesize} | ${latency}=${EMPTY}
 | | ...
@@ -449,7 +543,7 @@
 | | ...
 | | ... | *Example:*
 | | ...
-| | ... | \| Display result of NDRPDR search \| \${result} \| \${64}
+| | ... | \| Display result of NDRPDR search \| \${result} \| \${64} \|
 | | ...
 | | [Arguments] | ${result} | ${framesize}
 | | ...
@@ -477,15 +571,17 @@
 | | ... | *Example:*
 | | ...
 | | ... | \| Check NDRPDR interval validity \| \${result.pdr_interval} \
-| | ... | \| \${0.005}
+| | ... | \| \${0.005} \|
 | | ...
 | | [Arguments] | ${interval} | ${packet_loss_ratio}=${0.0}
 | | ...
-| | ${lower_bound_lf}= | Set Variable | ${interval.measured_low.loss_fraction}
+| | ${lower_bound_lf} = | Set Variable | ${interval.measured_low.loss_fraction}
 | | Return From Keyword If | ${lower_bound_lf} <= ${packet_loss_ratio}
-| | ${messagge}= | Catenate | SEPARATOR=${SPACE}
-| | ... | Lower bound fraction ${lower_bound_lf}
-| | ... | does not reach ${packet_loss_ratio}.
+| | ${message}= | Catenate | SEPARATOR=${SPACE}
+| | ... | Minimal rate loss fraction ${lower_bound_lf}
+| | ... | does not reach target ${packet_loss_ratio}.
+| | ${message} = | Set Variable If | ${lower_bound_lf} >= 1.0
+| | ... | ${message}${\n}Zero packets forwarded! | ${message}
 | | Fail | ${message}
 
 | Perform additional measurements based on NDRPDR result
@@ -503,7 +599,7 @@
 | | ...
 | | ... | *Example:*
 | | ... | \| Perform additional measurements based on NDRPDR result \
-| | ... | \| \${result} \| ${64} \| 3-node-IPv4
+| | ... | \| \${result} \| ${64} \| 3-node-IPv4 \|
 | | ...
 | | [Arguments] | ${result} | ${framesize} | ${topology_type}
 | | ...
@@ -673,7 +769,7 @@
 | | ... | *Example:*
 | | ...
 | | ... | \| Traffic should pass with maximum rate \| 4.0mpps \| 64 \
-| | ... | \| 3-node-IPv4 \| ${1} \| ${10.0} | ${False}
+| | ... | \| 3-node-IPv4 \| ${1} \| ${10.0} | ${False} \|
 | | ...
 | | [Arguments] | ${rate} | ${framesize} | ${topology_type}
 | | ... | ${trial_duration}=${perf_trial_duration} | ${fail_no_traffic}=${True}
@@ -703,7 +799,7 @@
 | | ... | *Example:*
 | | ...
 | | ... | \| Send traffic at specified rate \| ${1.0} \| 4.0mpps \| 64 \
-| | ... | \| 3-node-IPv4 \| ${10}
+| | ... | \| 3-node-IPv4 \| ${10} \|
 | | ...
 | | [Arguments] | ${trial_duration} | ${rate} | ${framesize}
 | | ... | ${topology_type} | ${subsamples}=${1}