DropRateSearch library

- linear search
- t-rex DropRateSearch implementation
- long perf bridge_domain test
- introduced PERFORMANCE_SHORT, PERFORMANCE_LONG test tags

Change-Id: I497b72f3e6d58a67ca5a386403d1e84dcf433ec4
Signed-off-by: Miroslav Miklus <mmiklus@cisco.com>

3 files changed, 407 insertions, 61 deletions

diff --git a/resources/libraries/python/DropRateSearch.py b/resources/libraries/python/DropRateSearch.py
new file mode 100644
index 0000000000..4bbf16c1a2
--- /dev/null
+++ b/resources/libraries/python/DropRateSearch.py
@@ -0,0 +1,301 @@
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Drop rate search algorithms"""
+
+from abc import ABCMeta, abstractmethod
+from enum import Enum, unique
+
+@unique
+class SearchDirection(Enum):
+    """Direction of linear search"""
+
+    TOP_DOWN = 1
+    BOTTOM_UP = 2
+
+@unique
+class SearchResults(Enum):
+    """Result of the drop rate search"""
+
+    SUCCESS = 1
+    FAILURE = 2
+    SUSPICIOUS = 3
+
+@unique
+class RateType(Enum):
+    """Type of rate units"""
+
+    PERCENTAGE = 1
+    PACKETS_PER_SECOND = 2
+    BITS_PER_SECOND = 3
+
+@unique
+class LossAcceptanceType(Enum):
+    """Type of the loss acceptance criteria"""
+
+    FRAMES = 1
+    PERCENTAGE = 2
+
+class DropRateSearch(object):
+    """Abstract class with search algorithm implementation"""
+
+    __metaclass__ = ABCMeta
+
+    def __init__(self):
+        #duration of traffic run (binary, linear)
+        self._duration = 60
+        #initial start rate (binary, linear)
+        self._rate_start = 100
+        #step of the linear search, unit: RateType (self._rate_type)
+        self._rate_linear_step = 10
+        #linear search direction, permitted values: SearchDirection
+        self._search_linear_direction = SearchDirection.TOP_DOWN
+        #upper limit of search, unit: RateType (self._rate_type)
+        self._rate_max = 100
+        #lower limit of search, unit: RateType (self._rate_type)
+        self._rate_min = 1
+        #permitted values: RateType
+        self._rate_type = RateType.PERCENTAGE
+        #accepted loss during search, units: LossAcceptanceType
+        self._loss_acceptance = 0
+        #permitted values: LossAcceptanceType
+        self._loss_acceptance_type = LossAcceptanceType.FRAMES
+        #size of frames to send
+        self._frame_size = "64"
+        #binary convergence criterium type is self._rate_type
+        self._binary_convergence_threshhold = "0.01"
+        #numbers of traffic runs during one rate step
+        self._max_attempts = 1
+
+        #result of search
+        self._search_result = None
+        self._search_result_rate = None
+
+    @abstractmethod
+    def measure_loss(self, rate, frame_size, loss_acceptance,
+                     loss_acceptance_type, traffic_type):
+        """Send traffic from TG and measure count of dropped frames
+
+        :param rate: offered traffic load
+        :param frame_size: size of frame
+        :param loss_acceptance: permitted drop ratio or frames count
+        :param loss_acceptance_type: type of permitted loss
+        :param traffic_type: traffic profile ([2,3]-node-L[2,3], ...)
+        :type rate: int
+        :type frame_size: str
+        :type loss_acceptance: float
+        :type loss_acceptance_type: LossAcceptanceType
+        :type traffic_type: str
+        :return: drop threshhold exceeded? (True/False)
+        :rtype bool
+        """
+        pass
+
+    def set_search_rate_boundaries(self, max_rate, min_rate):
+        """Set search boundaries: min,max
+
+        :param max_rate: upper value of search boundaries
+        :param min_rate: lower value of search boundaries
+        :type max_rate: float
+        :type min_rate: float
+        :return: nothing
+        """
+        if float(min_rate) <= 0:
+            raise ValueError("min_rate must be higher than 0")
+        elif float(min_rate) > float(max_rate):
+            raise ValueError("min_rate must be lower than max_rate")
+        else:
+            self._rate_max = float(max_rate)
+            self._rate_min = float(min_rate)
+
+    def set_search_linear_step(self, step_rate):
+        """Set step size for linear search
+
+        :param step_rate: linear search step size
+        :type step_rate: float
+        :return: nothing
+        """
+        self._rate_linear_step = float(step_rate)
+
+    def set_search_rate_type_percentage(self):
+        """Set rate type to percentage of linerate
+
+        :return: nothing
+        """
+        self._set_search_rate_type(RateType.PERCENTAGE)
+
+    def set_search_rate_type_bps(self):
+        """Set rate type to bits per second
+
+        :return: nothing
+        """
+        self._set_search_rate_type(RateType.BITS_PER_SECOND)
+
+    def set_search_rate_type_pps(self):
+        """Set rate type to packets per second
+
+        :return: nothing
+        """
+        self._set_search_rate_type(RateType.PACKETS_PER_SECOND)
+
+    def _set_search_rate_type(self, rate_type):
+        """Set rate type to one of RateType-s
+
+        :param rate_type: type of rate to set
+        :type rate_type: RateType
+        :return: nothing
+        """
+        if rate_type not in RateType:
+            raise Exception("rate_type unknown: {}".format(rate_type))
+        else:
+            self._rate_type = rate_type
+
+    def set_search_frame_size(self, frame_size):
+        """Set size of frames to send
+
+        :param frame_size: size of frames
+        :type frame_size: str
+        :return: nothing
+        """
+        self._frame_size = frame_size
+
+    def set_duration(self, duration):
+        """Set the duration of single traffic run
+
+        :param duration: number of seconds for traffic to run
+        :type duration: int
+        :return: nothing
+        """
+        self._duration = int(duration)
+
+    def get_duration(self):
+        """Return configured duration of single traffic run
+
+        :return: number of seconds for traffic to run
+        :rtype: int
+        """
+        return self._duration
+
+    def get_rate_type_str(self):
+        """Return rate type representation
+
+        :return: string representation of rate type
+        :rtype: str
+        """
+        if self._rate_type == RateType.PERCENTAGE:
+            return "%"
+        elif self._rate_type == RateType.BITS_PER_SECOND:
+            return "bps"
+        elif self._rate_type == RateType.PACKETS_PER_SECOND:
+            return "pps"
+        else:
+            raise ValueError("RateType unknown")
+
+    def linear_search(self, start_rate, traffic_type):
+        """Linear search of rate with loss below acceptance criteria
+
+        :param start_rate: initial rate
+        :param traffic_type: traffic profile
+        :type start_rate: float
+        :param traffic_type: str
+        :return: nothing
+        """
+
+        if not self._rate_min <= float(start_rate) <= self._rate_max:
+            raise ValueError("Start rate is not in min,max range")
+
+        rate = float(start_rate)
+        #the last but one step
+        prev_rate = None
+
+        #linear search
+        while True:
+            res = self.measure_loss(rate, self._frame_size,
+                                    self._loss_acceptance,
+                                    self._loss_acceptance_type,
+                                    traffic_type)
+            if self._search_linear_direction == SearchDirection.BOTTOM_UP:
+                #loss occured and it was above acceptance criteria
+                if res == False:
+                    #if this is first run then we didn't find drop rate
+                    if prev_rate == None:
+                        self._search_result = SearchResults.FAILURE
+                        self._search_result_rate = None
+                        return
+                    # else we found the rate, which is value from previous run
+                    else:
+                        self._search_result = SearchResults.SUCCESS
+                        self._search_result_rate = prev_rate
+                        return
+                #there was no loss / loss below acceptance criteria
+                elif res == True:
+                    prev_rate = rate
+                    rate += self._rate_linear_step
+                    if rate > self._rate_max:
+                        if prev_rate != self._rate_max:
+                            #one last step with rate set to _rate_max
+                            rate = self._rate_max
+                            continue
+                        else:
+                            self._search_result = SearchResults.SUCCESS
+                            self._search_result_rate = prev_rate
+                            return
+                    else:
+                        continue
+                else:
+                    raise RuntimeError("Unknown search result")
+
+            elif self._search_linear_direction == SearchDirection.TOP_DOWN:
+                #loss occured, decrease rate
+                if res == False:
+                    prev_rate = rate
+                    rate -= self._rate_linear_step
+                    if rate < self._rate_min:
+                        if prev_rate != self._rate_min:
+                            #one last step with rate set to _rate_min
+                            rate = self._rate_min
+                            continue
+                        else:
+                            self._search_result = SearchResults.FAILURE
+                            self._search_result_rate = None
+                            return
+                    else:
+                        continue
+                #no loss => non/partial drop rate found
+                elif res == True:
+                    self._search_result = SearchResults.SUCCESS
+                    self._search_result_rate = rate
+                    return
+                else:
+                    raise RuntimeError("Unknown search result")
+            else:
+                raise Exception("Unknown search direction")
+
+        raise Exception("Wrong codepath")
+
+    def verify_search_result(self):
+        """Fail if search was not successful
+
+        :return: result rate
+        :rtype: float
+        """
+        if self._search_result == SearchResults.FAILURE:
+            raise Exception('Search FAILED')
+        elif self._search_result in [SearchResults.SUCCESS, SearchResults.SUSPICIOUS]:
+            return self._search_result_rate
+
+    def binary_search(self):
+        raise NotImplementedError
+
+    def combined_search(self):
+        raise NotImplementedError
diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index fb9ae4db1c..c3af0863be 100644
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -14,17 +14,50 @@
 """Performance testing traffic generator library."""
 
 from robot.api import logger
+from robot.libraries.BuiltIn import BuiltIn
+from robot.api.deco import keyword
 
 from resources.libraries.python.ssh import SSH
 from resources.libraries.python.topology import NodeType
 from resources.libraries.python.topology import NodeSubTypeTG
 from resources.libraries.python.topology import Topology
+from resources.libraries.python.DropRateSearch import DropRateSearch
 
-__all__ = ['TrafficGenerator']
+__all__ = ['TrafficGenerator', 'TGDropRateSearchImpl']
+
+class TGDropRateSearchImpl(DropRateSearch):
+    """Drop Rate Search implementation"""
+
+    def __init__(self):
+        super(TGDropRateSearchImpl, self).__init__()
+
+    def measure_loss(self, rate, frame_size, loss_acceptance,
+                     loss_acceptance_type, traffic_type):
+
+        #we need instance of TrafficGenerator instantiated by Robot Framework
+        #to be able to use trex_stateless_remote_exec method
+        tg_instance = BuiltIn().get_library_instance('resources.libraries.python.TrafficGenerator')
+
+        if tg_instance._node['subtype'] is None:
+            raise Exception('TG subtype not defined')
+        elif tg_instance._node['subtype'] == NodeSubTypeTG.TREX:
+            unit_rate = str(rate) + self.get_rate_type_str()
+            tg_instance.trex_stateless_remote_exec(self.get_duration(), unit_rate,
+                                                   frame_size, traffic_type)
+
+            #TODO:getters for tg_instance and loss_acceptance_type
+            logger.trace("comparing: {} < {} ".format(tg_instance._loss, loss_acceptance))
+            if float(tg_instance._loss) > float(loss_acceptance):
+                return False
+            else:
+                return True
+        else:
+            raise NotImplementedError("TG subtype not supported")
 
 class TrafficGenerator(object):
     """Traffic Generator"""
 
+    #use one instance of TrafficGenerator for all tests in test suite
     ROBOT_LIBRARY_SCOPE = 'TEST SUITE'
 
     def __init__(self):
@@ -32,6 +65,7 @@ class TrafficGenerator(object):
         self._loss = None
         self._sent = None
         self._received = None
+        self._node = None
         #T-REX interface order mapping
         self._ifaces_reordered = 0
 
@@ -69,6 +103,8 @@ class TrafficGenerator(object):
 
         if tg_node['type'] != NodeType.TG:
             raise Exception('Node type is not a TG')
+        self._node = tg_node
+
         if tg_node['subtype'] == NodeSubTypeTG.TREX:
             ssh = SSH()
             ssh.connect(tg_node)
@@ -103,6 +139,7 @@ class TrafficGenerator(object):
                 "- port_limit      : 2\n"
                 "  version         : 2\n"
                 "  interfaces      : [\"{}\",\"{}\"]\n"
+                "  port_bandwidth_gb : 10\n"
                 "  port_info       :\n"
                 "          - dest_mac        :   [{}]\n"
                 "            src_mac         :   [{}]\n"
@@ -152,16 +189,76 @@ class TrafficGenerator(object):
                 logger.error('pkill t-rex failed: {0}'.format(stdout + stderr))
                 raise RuntimeError('pkill t-rex failed')
 
-    def send_traffic_on(self, nodes_info, duration, rate,
+    def trex_stateless_remote_exec(self, duration, rate, framesize,
+                                   traffic_type):
+        """Execute stateless script on remote node over ssh
+
+        :param node: remote node
+        :param traffic_type: Traffic profile
+        :type node: dict
+        :type traffic_type: str
+        """
+        ssh = SSH()
+        ssh.connect(self._node)
+
+        _p0 = 1
+        _p1 = 2
+
+        if self._ifaces_reordered != 0:
+            _p0, _p1 = _p1, _p0
+
+        if traffic_type in ["3-node-xconnect", "3-node-bridge"]:
+            (ret, stdout, stderr) = ssh.exec_command(
+                "sh -c '/tmp/openvpp-testing/resources/tools/t-rex/"
+                "t-rex-stateless.py "
+                "-d {0} -r {1} -s {2} "
+                "--p{3}_src_start_ip 10.10.10.1 "
+                "--p{3}_src_end_ip 10.10.10.254 "
+                "--p{3}_dst_start_ip 20.20.20.1 "
+                "--p{4}_src_start_ip 20.20.20.1 "
+                "--p{4}_src_end_ip 20.20.20.254 "
+                "--p{4}_dst_start_ip 10.10.10.1'".\
+                format(duration, rate, framesize, _p0, _p1),\
+                timeout=int(duration)+60)
+        elif traffic_type in ["3-node-IPv4"]:
+            (ret, stdout, stderr) = ssh.exec_command(
+                "sh -c '/tmp/openvpp-testing/resources/tools/t-rex/"
+                "t-rex-stateless.py "
+                "-d {0} -r {1} -s {2} "
+                "--p{3}_src_start_ip 10.10.10.2 "
+                "--p{3}_src_end_ip 10.10.10.254 "
+                "--p{3}_dst_start_ip 20.20.20.2 "
+                "--p{4}_src_start_ip 20.20.20.2 "
+                "--p{4}_src_end_ip 20.20.20.254 "
+                "--p{4}_dst_start_ip 10.10.10.2'".\
+                format(duration, rate, framesize, _p0, _p1),\
+                timeout=int(duration)+60)
+        else:
+            raise NotImplementedError('Unsupported traffic type')
+
+        logger.trace(ret)
+        logger.trace(stdout)
+        logger.trace(stderr)
+
+        #last line from console output
+        line = stdout.splitlines()[-1]
+
+        self._result = line
+        logger.info('TrafficGen result: {0}'.format(self._result))
+
+        self._received = self._result.split(', ')[1].split('=')[1]
+        self._sent = self._result.split(', ')[2].split('=')[1]
+        self._loss = self._result.split(', ')[3].split('=')[1]
+
+    def send_traffic_on(self, node, duration, rate,
                         framesize, traffic_type):
         """Send traffic from all configured interfaces on TG
-        :param nodes_info: Dictionary containing information on all nodes
-        in topology.
+        :param node: Dictionary containing TG information
         :param duration: Duration of test traffic generation in seconds
         :param rate: Offered load per interface (e.g. 1%, 3gbps, 4mpps, ...)
         :param framesize: Frame size (L2) in Bytes
         :param traffic_type: Traffic profile
-        :type nodes_info: dict
+        :type node: dict
         :type duration: str
         :type rate: str
         :type framesize: str
@@ -170,69 +267,17 @@ class TrafficGenerator(object):
         :rtype: str
         """
 
-        node = nodes_info["TG"]
-
         if node['type'] != NodeType.TG:
             raise Exception('Node type is not a TG')
 
         if node['subtype'] is None:
             raise Exception('TG subtype not defined')
-
-        ssh = SSH()
-        ssh.connect(node)
-
-        if node['subtype'] == NodeSubTypeTG.TREX:
-
-            _p0 = 1
-            _p1 = 2
-
-            if self._ifaces_reordered != 0:
-                _p0, _p1 = _p1, _p0
-
-            if traffic_type in ["3-node-xconnect", "3-node-bridge"]:
-                (ret, stdout, stderr) = ssh.exec_command(
-                    "sh -c '/tmp/openvpp-testing/resources/tools/t-rex/"
-                    "t-rex-stateless.py "
-                    "-d {0} -r {1} -s {2} "
-                    "--p{3}_src_start_ip 10.10.10.1 "
-                    "--p{3}_src_end_ip 10.10.10.254 "
-                    "--p{3}_dst_start_ip 20.20.20.1 "
-                    "--p{4}_src_start_ip 20.20.20.1 "
-                    "--p{4}_src_end_ip 20.20.20.254 "
-                    "--p{4}_dst_start_ip 10.10.10.1'".\
-                    format(duration, rate, framesize, _p0, _p1),\
-                    timeout=int(duration)+60)
-            elif traffic_type in ["3-node-IPv4"]:
-                (ret, stdout, stderr) = ssh.exec_command(
-                    "sh -c '/tmp/openvpp-testing/resources/tools/t-rex/"
-                    "t-rex-stateless.py "
-                    "-d {0} -r {1} -s {2} "
-                    "--p{3}_src_start_ip 10.10.10.2 "
-                    "--p{3}_src_end_ip 10.10.10.254 "
-                    "--p{3}_dst_start_ip 20.20.20.2 "
-                    "--p{4}_src_start_ip 20.20.20.2 "
-                    "--p{4}_src_end_ip 20.20.20.254 "
-                    "--p{4}_dst_start_ip 10.10.10.2'".\
-                    format(duration, rate, framesize, _p0, _p1),\
-                    timeout=int(duration)+60)
-            else:
-                raise NotImplementedError('Unsupported traffic type')
-
+        elif node['subtype'] == NodeSubTypeTG.TREX:
+            self.trex_stateless_remote_exec(duration, rate, framesize,
+                                            traffic_type)
         else:
             raise NotImplementedError("TG subtype not supported")
 
-        logger.trace(ret)
-        logger.trace(stdout)
-        logger.trace(stderr)
-
-        for line in stdout.splitlines():
-            pass
-
-        self._result = line
-        logger.info('TrafficGen result: {0}'.format(self._result))
-
-        self._loss = self._result.split(', ')[3].split('=')[1]
-
         return self._result
 
     def no_traffic_loss_occured(self):
diff --git a/resources/tools/t-rex/t-rex-stateless.py b/resources/tools/t-rex/t-rex-stateless.py
index 6c53794739..46c67cc829 100755
--- a/resources/tools/t-rex/t-rex-stateless.py
+++ b/resources/tools/t-rex/t-rex-stateless.py
@@ -158,7 +158,7 @@ def simple_burst(pkt_a, pkt_b, duration=10, rate="1mpps",
         c.start(ports=[0, 1], mult=rate, duration=duration)
 
         # block until done
-        c.wait_on_traffic(ports=[0, 1])
+        c.wait_on_traffic(ports=[0, 1], timeout=(duration+30))
 
         # read the stats after the test
         stats = c.get_stats()