Python3: resources and libraries

Change-Id: I1392c06b1d64f62b141d24c0d42a8e36913b15e2
Signed-off-by: Jan Gelety <jgelety@cisco.com>

3 files changed, 192 insertions, 169 deletions

diff --git a/resources/tools/wrk/wrk.py b/resources/tools/wrk/wrk.py
index 84d17ee7a1..381e9b9da0 100644
--- a/resources/tools/wrk/wrk.py
+++ b/resources/tools/wrk/wrk.py
@@ -66,18 +66,18 @@ def check_wrk(tg_node):
         command is not availble.
     """
 
-    if tg_node['type'] != NodeType.TG:
-        raise RuntimeError('Node type is not a TG.')
+    if tg_node[u"type"] != NodeType.TG:
+        raise RuntimeError(u"Node type is not a TG.")
 
     ssh = SSH()
     ssh.connect(tg_node)
 
     ret, _, _ = ssh.exec_command(
-        "sudo -E "
-        "sh -c '{0}/resources/tools/wrk/wrk_utils.sh installed'".
-        format(Constants.REMOTE_FW_DIR))
+        f"sudo -E sh -c '{Constants.REMOTE_FW_DIR}/resources/tools/"
+        f"wrk/wrk_utils.sh installed'"
+    )
     if int(ret) != 0:
-        raise RuntimeError('WRK is not installed on TG node.')
+        raise RuntimeError(u"WRK is not installed on TG node.")
 
 
 def run_wrk(tg_node, profile_name, tg_numa, test_type, warm_up=False):
@@ -98,102 +98,103 @@ def run_wrk(tg_node, profile_name, tg_numa, test_type, warm_up=False):
     :raises: RuntimeError if node type is not a TG.
     """
 
-    if tg_node['type'] != NodeType.TG:
-        raise RuntimeError('Node type is not a TG.')
+    if tg_node[u"type"] != NodeType.TG:
+        raise RuntimeError(u"Node type is not a TG.")
 
     # Parse and validate the profile
-    profile_path = ("resources/traffic_profiles/wrk/{0}.yaml".
-                    format(profile_name))
+    profile_path = f"resources/traffic_profiles/wrk/{profile_name}.yaml"
     profile = WrkTrafficProfile(profile_path).traffic_profile
 
     cores = CpuUtils.cpu_list_per_node(tg_node, tg_numa)
-    first_cpu = cores[profile["first-cpu"]]
+    first_cpu = cores[profile[u"first-cpu"]]
 
-    if len(profile["urls"]) == 1 and profile["cpus"] == 1:
+    if len(profile[u"urls"]) == 1 and profile[u"cpus"] == 1:
         params = [
-            "traffic_1_url_1_core",
+            u"traffic_1_url_1_core",
             str(first_cpu),
-            str(profile["nr-of-threads"]),
-            str(profile["nr-of-connections"]),
-            "{0}s".format(profile["duration"]),
-            "'{0}'".format(profile["header"]),
-            str(profile["timeout"]),
-            str(profile["script"]),
-            str(profile["latency"]),
-            "'{0}'".format(" ".join(profile["urls"]))
+            str(profile[u"nr-of-threads"]),
+            str(profile[u"nr-of-connections"]),
+            f"{profile[u'duration']}s",
+            f"'{profile[u'header']}'",
+            str(profile[u"timeout"]),
+            str(profile[u"script"]),
+            str(profile[u"latency"]),
+            f"'{u' '.join(profile[u'urls'])}'"
         ]
         if warm_up:
             warm_up_params = deepcopy(params)
-            warm_up_params[4] = "10s"
-    elif len(profile["urls"]) == profile["cpus"]:
+            warm_up_params[4] = u"10s"
+    elif len(profile[u"urls"]) == profile[u"cpus"]:
         params = [
-            "traffic_n_urls_n_cores",
+            u"traffic_n_urls_n_cores",
             str(first_cpu),
-            str(profile["nr-of-threads"]),
-            str(profile["nr-of-connections"]),
-            "{0}s".format(profile["duration"]),
-            "'{0}'".format(profile["header"]),
-            str(profile["timeout"]),
-            str(profile["script"]),
-            str(profile["latency"]),
-            "'{0}'".format(" ".join(profile["urls"]))
+            str(profile[u"nr-of-threads"]),
+            str(profile[u"nr-of-connections"]),
+            f"{profile[u'duration']}s",
+            f"'{profile[u'header']}'",
+            str(profile[u"timeout"]),
+            str(profile[u"script"]),
+            str(profile[u"latency"]),
+            f"'{u' '.join(profile[u'urls'])}'"
         ]
         if warm_up:
             warm_up_params = deepcopy(params)
-            warm_up_params[4] = "10s"
+            warm_up_params[4] = u"10s"
     else:
         params = [
-            "traffic_n_urls_m_cores",
+            u"traffic_n_urls_m_cores",
             str(first_cpu),
-            str(profile["cpus"] / len(profile["urls"])),
-            str(profile["nr-of-threads"]),
-            str(profile["nr-of-connections"]),
-            "{0}s".format(profile["duration"]),
-            "'{0}'".format(profile["header"]),
-            str(profile["timeout"]),
-            str(profile["script"]),
-            str(profile["latency"]),
-            "'{0}'".format(" ".join(profile["urls"]))
+            str(profile[u"cpus"] // len(profile[u"urls"])),
+            str(profile[u"nr-of-threads"]),
+            str(profile[u"nr-of-connections"]),
+            f"{profile[u'duration']}s",
+            f"'{profile[u'header']}'",
+            str(profile[u"timeout"]),
+            str(profile[u"script"]),
+            str(profile[u"latency"]),
+            f"'{u' '.join(profile[u'urls'])}'"
         ]
         if warm_up:
             warm_up_params = deepcopy(params)
-            warm_up_params[5] = "10s"
+            warm_up_params[5] = u"10s"
 
-    args = " ".join(params)
+    args = u" ".join(params)
 
     ssh = SSH()
     ssh.connect(tg_node)
 
     if warm_up:
-        warm_up_args = " ".join(warm_up_params)
+        warm_up_args = u" ".join(warm_up_params)
         ret, _, _ = ssh.exec_command(
-            "{0}/resources/tools/wrk/wrk_utils.sh {1}".
-            format(Constants.REMOTE_FW_DIR, warm_up_args), timeout=1800)
+            f"{Constants.REMOTE_FW_DIR}/resources/tools/wrk/wrk_utils.sh "
+            f"{warm_up_args}", timeout=1800
+        )
         if int(ret) != 0:
-            raise RuntimeError('wrk runtime error.')
+            raise RuntimeError(u"wrk runtime error.")
         sleep(60)
 
     ret, stdout, _ = ssh.exec_command(
-        "{0}/resources/tools/wrk/wrk_utils.sh {1}".
-        format(Constants.REMOTE_FW_DIR, args), timeout=1800)
+        f"{Constants.REMOTE_FW_DIR}/resources/tools/wrk/wrk_utils.sh {args}",
+        timeout=1800
+    )
     if int(ret) != 0:
         raise RuntimeError('wrk runtime error.')
 
     stats = _parse_wrk_output(stdout)
 
-    log_msg = "\nMeasured values:\n"
-    if test_type == "cps":
-        log_msg += "Connections/sec: Avg / Stdev / Max  / +/- Stdev\n"
-        for item in stats["rps-stats-lst"]:
-            log_msg += "{0} / {1} / {2} / {3}\n".format(*item)
-        log_msg += "Total cps: {0}cps\n".format(stats["rps-sum"])
-    elif test_type == "rps":
-        log_msg += "Requests/sec: Avg / Stdev / Max  / +/- Stdev\n"
-        for item in stats["rps-stats-lst"]:
-            log_msg += "{0} / {1} / {2} / {3}\n".format(*item)
-        log_msg += "Total rps: {0}rps\n".format(stats["rps-sum"])
-    elif test_type == "bw":
-        log_msg += "Transfer/sec: {0}Bps".format(stats["bw-sum"])
+    log_msg = u"\nMeasured values:\n"
+    if test_type == u"cps":
+        log_msg += u"Connections/sec: Avg / Stdev / Max  / +/- Stdev\n"
+        for item in stats[u"rps-stats-lst"]:
+            log_msg += f"{0} / {1} / {2} / {3}\n".format(*item)
+        log_msg += f"Total cps: {stats[u'rps-sum']}cps\n"
+    elif test_type == u"rps":
+        log_msg += u"Requests/sec: Avg / Stdev / Max  / +/- Stdev\n"
+        for item in stats[u"rps-stats-lst"]:
+            log_msg += f"{0} / {1} / {2} / {3}\n".format(*item)
+        log_msg += f"Total rps: {stats[u'rps-sum']}rps\n"
+    elif test_type == u"bw":
+        log_msg += f"Transfer/sec: {stats[u'bw-sum']}Bps"
 
     logger.info(log_msg)
 
@@ -210,47 +211,52 @@ def _parse_wrk_output(msg):
     :raises: WrkError if the message does not include the results.
     """
 
-    if "Thread Stats" not in msg:
-        raise WrkError("The output of wrk does not include the results.")
+    if u"Thread Stats" not in msg:
+        raise WrkError(u"The output of wrk does not include the results.")
 
     msg_lst = msg.splitlines(False)
 
     stats = {
-        "latency-dist-lst": list(),
-        "latency-stats-lst": list(),
-        "rps-stats-lst": list(),
-        "rps-lst": list(),
-        "bw-lst": list(),
-        "rps-sum": 0,
-        "bw-sum": None
+        u"latency-dist-lst": list(),
+        u"latency-stats-lst": list(),
+        u"rps-stats-lst": list(),
+        u"rps-lst": list(),
+        u"bw-lst": list(),
+        u"rps-sum": 0,
+        u"bw-sum": None
     }
 
     for line in msg_lst:
-        if "Latency Distribution" in line:
+        if u"Latency Distribution" in line:
             # Latency distribution - 50%, 75%, 90%, 99%
             pass
-        elif "Latency" in line:
+        elif u"Latency" in line:
             # Latency statistics - Avg, Stdev, Max, +/- Stdev
             pass
-        elif "Req/Sec" in line:
+        elif u"Req/Sec" in line:
             # rps statistics - Avg, Stdev, Max, +/- Stdev
-            stats["rps-stats-lst"].append((
-                _evaluate_number(re.search(REGEX_RPS_STATS, line).group(1)),
-                _evaluate_number(re.search(REGEX_RPS_STATS, line).group(2)),
-                _evaluate_number(re.search(REGEX_RPS_STATS, line).group(3)),
-                _evaluate_number(re.search(REGEX_RPS_STATS, line).group(4))))
-        elif "Requests/sec:" in line:
+            stats[u"rps-stats-lst"].append(
+                (
+                    _evaluate_number(re.search(REGEX_RPS_STATS, line).group(1)),
+                    _evaluate_number(re.search(REGEX_RPS_STATS, line).group(2)),
+                    _evaluate_number(re.search(REGEX_RPS_STATS, line).group(3)),
+                    _evaluate_number(re.search(REGEX_RPS_STATS, line).group(4))
+                )
+            )
+        elif u"Requests/sec:" in line:
             # rps (cps)
-            stats["rps-lst"].append(
-                _evaluate_number(re.search(REGEX_RPS, line).group(1)))
-        elif "Transfer/sec:" in line:
+            stats[u"rps-lst"].append(
+                _evaluate_number(re.search(REGEX_RPS, line).group(1))
+            )
+        elif u"Transfer/sec:" in line:
             # BW
-            stats["bw-lst"].append(
-                _evaluate_number(re.search(REGEX_BW, line).group(1)))
+            stats[u"bw-lst"].append(
+                _evaluate_number(re.search(REGEX_BW, line).group(1))
+            )
 
-    for item in stats["rps-stats-lst"]:
-        stats["rps-sum"] += item[0]
-    stats["bw-sum"] = sum(stats["bw-lst"])
+    for item in stats[u"rps-stats-lst"]:
+        stats[u"rps-sum"] += item[0]
+    stats[u"bw-sum"] = sum(stats[u"bw-lst"])
 
     return stats
 
@@ -270,23 +276,24 @@ def _evaluate_number(num):
     try:
         val_num = float(val.group(1))
     except ValueError:
-        raise WrkError("The output of wrk does not include the results "
-                       "or the format of results has changed.")
+        raise WrkError(
+            u"The output of wrk does not include the results or the format "
+            u"of results has changed."
+        )
     val_mul = val.group(2).lower()
     if val_mul:
-        if "k" in val_mul:
+        if u"k" in val_mul:
             val_num *= 1000
-        elif "m" in val_mul:
+        elif u"m" in val_mul:
             val_num *= 1000000
-        elif "g" in val_mul:
+        elif u"g" in val_mul:
             val_num *= 1000000000
-        elif "b" in val_mul:
+        elif u"b" in val_mul:
             pass
-        elif "%" in val_mul:
+        elif u"%" in val_mul:
             pass
-        elif "" in val_mul:
+        elif u"" in val_mul:
             pass
         else:
-            raise WrkError("The multiplicand {0} is not defined.".
-                           format(val_mul))
+            raise WrkError(f"The multiplicand {val_mul} is not defined.")
     return val_num
diff --git a/resources/tools/wrk/wrk_errors.py b/resources/tools/wrk/wrk_errors.py
index 3173dd4223..2cdd76815a 100644
--- a/resources/tools/wrk/wrk_errors.py
+++ b/resources/tools/wrk/wrk_errors.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -29,7 +29,7 @@ class WrkError(Exception):
      - relevant data if there are any collected (optional parameter details).
     """
 
-    def __init__(self, msg, details=''):
+    def __init__(self, msg, details=u""):
         """Sets the exception message and the level.
 
         :param msg: Short description of the encountered problem.
diff --git a/resources/tools/wrk/wrk_traffic_profile_parser.py b/resources/tools/wrk/wrk_traffic_profile_parser.py
index 1d40aa3d8a..1994b6195d 100644
--- a/resources/tools/wrk/wrk_traffic_profile_parser.py
+++ b/resources/tools/wrk/wrk_traffic_profile_parser.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and / or its affiliates.
+# Copyright (c) 2019 Cisco and / or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
 # use this file except in compliance with the License. You may obtain a copy
 # of the License at:
@@ -26,22 +26,26 @@ from robot.api import logger
 from resources.tools.wrk.wrk_errors import WrkError
 
 
-class WrkTrafficProfile(object):
+class WrkTrafficProfile:
     """The wrk traffic profile.
     """
 
-    MANDATORY_PARAMS = ("urls",
-                        "first-cpu",
-                        "cpus",
-                        "duration",
-                        "nr-of-threads",
-                        "nr-of-connections")
-
-    INTEGER_PARAMS = (("cpus", 1),
-                      ("first-cpu", 0),
-                      ("duration", 1),
-                      ("nr-of-threads", 1),
-                      ("nr-of-connections", 1))
+    MANDATORY_PARAMS = (
+        u"urls",
+        u"first-cpu",
+        u"cpus",
+        u"duration",
+        u"nr-of-threads",
+        u"nr-of-connections"
+    )
+
+    INTEGER_PARAMS = (
+        (u"cpus", 1),
+        (u"first-cpu", 0),
+        (u"duration", 1),
+        (u"nr-of-threads", 1),
+        (u"nr-of-connections", 1)
+    )
 
     def __init__(self, profile_name):
         """Read the traffic profile from the yaml file.
@@ -57,29 +61,34 @@ class WrkTrafficProfile(object):
         self.profile_name = profile_name
 
         try:
-            with open(self.profile_name, 'r') as profile_file:
+            with open(self.profile_name, "r") as profile_file:
                 self.traffic_profile = load(profile_file)
         except IOError as err:
-            raise WrkError(msg="An error occurred while opening the file '{0}'."
-                           .format(self.profile_name),
-                           details=str(err))
+            raise WrkError(
+                msg=f"An error occurred while opening the file "
+                f"'{self.profile_name}'.", details=str(err)
+            )
         except YAMLError as err:
-            raise WrkError(msg="An error occurred while parsing the traffic "
-                               "profile '{0}'.".format(self.profile_name),
-                           details=str(err))
+            raise WrkError(
+                msg=f"An error occurred while parsing the traffic profile "
+                f"'{self.profile_name}'.", details=str(err)
+            )
 
         self._validate_traffic_profile()
 
         if self.traffic_profile:
-            logger.debug("\nThe wrk traffic profile '{0}' is valid.\n".
-                         format(self.profile_name))
-            logger.debug("wrk traffic profile '{0}':".format(self.profile_name))
+            logger.debug(
+                f"\nThe wrk traffic profile '{self.profile_name}' is valid.\n"
+            )
+            logger.debug(f"wrk traffic profile '{self.profile_name}':")
             logger.debug(pformat(self.traffic_profile))
         else:
-            logger.debug("\nThe wrk traffic profile '{0}' is invalid.\n".
-                         format(self.profile_name))
-            raise WrkError("\nThe wrk traffic profile '{0}' is invalid.\n".
-                           format(self.profile_name))
+            logger.debug(
+                f"\nThe wrk traffic profile '{self.profile_name}' is invalid.\n"
+            )
+            raise WrkError(
+                f"\nThe wrk traffic profile '{self.profile_name}' is invalid.\n"
+            )
 
     def __repr__(self):
         return pformat(self.traffic_profile)
@@ -94,8 +103,9 @@ class WrkTrafficProfile(object):
         doc/wrk_lld.rst
         """
 
-        logger.debug("\nValidating the wrk traffic profile '{0}'...\n".
-                     format(self.profile_name))
+        logger.debug(
+            f"\nValidating the wrk traffic profile '{self.profile_name}'...\n"
+        )
         if not (self._validate_mandatory_structure()
                 and self._validate_mandatory_values()
                 and self._validate_optional_values()
@@ -110,14 +120,14 @@ class WrkTrafficProfile(object):
         """
         # Level 1: Check if the profile is a dictionary:
         if not isinstance(self.traffic_profile, dict):
-            logger.error("The wrk traffic profile must be a dictionary.")
+            logger.error(u"The wrk traffic profile must be a dictionary.")
             return False
 
         # Level 2: Check if all mandatory parameters are present:
         is_valid = True
         for param in self.MANDATORY_PARAMS:
             if self.traffic_profile.get(param, None) is None:
-                logger.error("The parameter '{0}' in mandatory.".format(param))
+                logger.error(f"The parameter '{param}' in mandatory.")
                 is_valid = False
         return is_valid
 
@@ -129,8 +139,8 @@ class WrkTrafficProfile(object):
         """
         # Level 3: Mandatory params: Check if urls is a list:
         is_valid = True
-        if not isinstance(self.traffic_profile["urls"], list):
-            logger.error("The parameter 'urls' must be a list.")
+        if not isinstance(self.traffic_profile[u"urls"], list):
+            logger.error(u"The parameter 'urls' must be a list.")
             is_valid = False
 
         # Level 3: Mandatory params: Check if integers are not below minimum
@@ -147,60 +157,64 @@ class WrkTrafficProfile(object):
         """
         is_valid = True
         # Level 4: Optional params: Check if script is present:
-        script = self.traffic_profile.get("script", None)
+        script = self.traffic_profile.get(u"script", None)
         if script is not None:
             if not isinstance(script, str):
-                logger.error("The path to LuaJIT script in invalid")
+                logger.error(u"The path to LuaJIT script in invalid")
                 is_valid = False
             else:
                 if not isfile(script):
-                    logger.error("The file '{0}' does not exist.".
-                                 format(script))
+                    logger.error(f"The file '{script}' does not exist.")
                     is_valid = False
         else:
-            self.traffic_profile["script"] = None
-            logger.debug("The optional parameter 'LuaJIT script' is not "
-                         "defined. No problem.")
+            self.traffic_profile[u"script"] = None
+            logger.debug(
+                u"The optional parameter 'LuaJIT script' is not defined. "
+                u"No problem."
+            )
 
         # Level 4: Optional params: Check if header is present:
-        header = self.traffic_profile.get("header", None)
+        header = self.traffic_profile.get(u"header", None)
         if header is not None:
             if isinstance(header, dict):
-                header = ", ".join("{0}: {1}".format(*item)
-                                   for item in header.items())
-                self.traffic_profile["header"] = header
+                header = u", ".join(
+                    f"{0}: {1}".format(*item) for item in header.items()
+                )
+                self.traffic_profile[u"header"] = header
             elif not isinstance(header, str):
-                logger.error("The parameter 'header' type is not valid.")
+                logger.error(u"The parameter 'header' type is not valid.")
                 is_valid = False
 
             if not header:
-                logger.error("The parameter 'header' is defined but "
-                             "empty.")
+                logger.error(u"The parameter 'header' is defined but empty.")
                 is_valid = False
         else:
-            self.traffic_profile["header"] = None
-            logger.debug("The optional parameter 'header' is not defined. "
-                         "No problem.")
+            self.traffic_profile[u"header"] = None
+            logger.debug(
+                u"The optional parameter 'header' is not defined. No problem."
+            )
 
         # Level 4: Optional params: Check if latency is present:
-        latency = self.traffic_profile.get("latency", None)
+        latency = self.traffic_profile.get(u"latency", None)
         if latency is not None:
             if not isinstance(latency, bool):
-                logger.error("The parameter 'latency' must be boolean.")
+                logger.error(u"The parameter 'latency' must be boolean.")
                 is_valid = False
         else:
-            self.traffic_profile["latency"] = False
-            logger.debug("The optional parameter 'latency' is not defined. "
-                         "No problem.")
+            self.traffic_profile[u"latency"] = False
+            logger.debug(
+                u"The optional parameter 'latency' is not defined. No problem."
+            )
 
         # Level 4: Optional params: Check if timeout is present:
-        if 'timeout' in self.traffic_profile:
-            if not self._validate_int_param('timeout', 1):
+        if u"timeout" in self.traffic_profile:
+            if not self._validate_int_param(u"timeout", 1):
                 is_valid = False
         else:
-            self.traffic_profile["timeout"] = None
-            logger.debug("The optional parameter 'timeout' is not defined. "
-                         "No problem.")
+            self.traffic_profile[u"timeout"] = None
+            logger.debug(
+                u"The optional parameter 'timeout' is not defined. No problem."
+            )
 
         return is_valid
 
@@ -211,9 +225,10 @@ class WrkTrafficProfile(object):
         :rtype: bool
         """
         # Level 5: Check urls and cpus:
-        if self.traffic_profile["cpus"] % len(self.traffic_profile["urls"]):
-            logger.error("The number of CPUs must be a multiple of the "
-                         "number of URLs.")
+        if self.traffic_profile[u"cpus"] % len(self.traffic_profile[u"urls"]):
+            logger.error(
+                u"The number of CPUs must be a multiple of the number of URLs."
+            )
             return False
         return True
 
@@ -229,7 +244,7 @@ class WrkTrafficProfile(object):
         :rtype: bool
         """
         value = self._traffic_profile[param]
-        if isinstance(value, (str, unicode)):
+        if isinstance(value, str):
             if value.isdigit():
                 value = int(value)
             else:
@@ -237,8 +252,9 @@ class WrkTrafficProfile(object):
         if isinstance(value, int) and value >= minimum:
             self.traffic_profile[param] = value
             return True
-        logger.error("The parameter '{param}' must be an integer and "
-                     "at least {minimum}".format(param=param, minimum=minimum))
+        logger.error(
+            f"The parameter '{param}' must be an integer and at least {minimum}"
+        )
         return False
 
     @property