Python 3 - another drop (package fixups)

7 files changed, 88 insertions, 68 deletions

diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py
index 9d761156..fe4bad62 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_ext.py
@@ -16,8 +16,7 @@ if not TREX_STL_EXT_PATH:
 # the modules required
 # py-dep requires python2/python3 directories
 # arch-dep requires cel59/fedora and 32bit/64bit directories
-CLIENT_UTILS_MODULES = [ {'name': 'dpkt-1.8.6'},
-                         {'name': 'texttable-0.8.4'},
+CLIENT_UTILS_MODULES = [ {'name': 'texttable-0.8.4'},
                          {'name': 'pyyaml-3.11', 'py-dep': True},
                          {'name': 'scapy-2.3.1', 'py-dep': True},
                          {'name': 'pyzmq-14.5.0', 'py-dep': True, 'arch-dep': True}
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
index 23ecaf83..9387c3a6 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
@@ -172,11 +172,12 @@ import sys
 import os
 import socket
 import copy
-from trex_stl_lib.api import *
-from trex_stl_types import *
-from utils.common import get_number
 from collections import defaultdict
 
+from trex_stl_lib.api import *
+from .trex_stl_types import *
+from .utils.common import get_number
+
 
 class HLT_ERR(dict):
     def __init__(self, log = 'Unknown error', **kwargs):
@@ -877,11 +878,15 @@ def generate_packet(**user_kwargs):
                 raise STLError('mac_src_count has to be at least 1')
             if count > 0 or kwargs['mac_src_mode'] == 'random':
                 mac_src = ipv4_str_to_num(mac2str(kwargs['mac_src'])[2:]) # take only 32 lsb
+
                 step = kwargs['mac_src_step']
-                if step < 1:
-                    raise STLError('mac_src_step has to be at least 1')
+
                 if type(step) is str:
                     step = ipv4_str_to_num(mac2str(step)[2:]) # take only 32 lsb
+
+                if step < 1:
+                    raise STLError('mac_src_step has to be at least 1')
+                
                 if kwargs['mac_src_mode'] == 'increment':
                     add_val = mac_src - 0x7fffffff
                     var_name = '%s_%s_%s_%s' % ('inc', 4, count, step)
@@ -913,10 +918,13 @@ def generate_packet(**user_kwargs):
             if count > 0 or kwargs['mac_dst_mode'] == 'random':
                 mac_dst = ipv4_str_to_num(mac2str(kwargs['mac_dst'])[2:]) # take only 32 lsb
                 step = kwargs['mac_dst_step']
-                if step < 1:
-                    raise STLError('mac_dst_step has to be at least 1')
+
                 if type(step) is str:
                     step = ipv4_str_to_num(mac2str(step)[2:]) # take only 32 lsb
+
+                if step < 1:
+                    raise STLError('mac_dst_step has to be at least 1')
+                
                 if kwargs['mac_dst_mode'] == 'increment':
                     add_val = mac_dst - 0x7fffffff
                     var_name = '%s_%s_%s_%s' % ('inc', 4, count, step)
@@ -1037,10 +1045,12 @@ def generate_packet(**user_kwargs):
                 if type(ip_src_addr) is str:
                     ip_src_addr = ipv4_str_to_num(is_valid_ipv4(ip_src_addr))
                 step = kwargs['ip_src_step']
-                if step < 1:
-                    raise STLError('ip_src_step has to be at least 1')
                 if type(step) is str:
                     step = ipv4_str_to_num(is_valid_ipv4(step))
+
+                if step < 1:
+                    raise STLError('ip_src_step has to be at least 1')
+                
                 if kwargs['ip_src_mode'] == 'increment':
                     add_val = ip_src_addr - 0x7fffffff
                     var_name = '%s_%s_%s_%s' % ('inc', 4, count, step)
@@ -1075,10 +1085,13 @@ def generate_packet(**user_kwargs):
                 if type(ip_dst_addr) is str:
                     ip_dst_addr = ipv4_str_to_num(is_valid_ipv4(ip_dst_addr))
                 step = kwargs['ip_dst_step']
-                if step < 1:
-                    raise STLError('ip_dst_step has to be at least 1')
+
                 if type(step) is str:
                     step = ipv4_str_to_num(is_valid_ipv4(step))
+
+                if step < 1:
+                    raise STLError('ip_dst_step has to be at least 1')
+                
                 if kwargs['ip_dst_mode'] == 'increment':
                     add_val = ip_dst_addr - 0x7fffffff
                     var_name = '%s_%s_%s_%s' % ('inc', 4, count, step)
@@ -1131,10 +1144,13 @@ def generate_packet(**user_kwargs):
             if count > 0 or kwargs['ipv6_src_mode'] == 'random':
                 ipv6_src_addr_num = ipv4_str_to_num(is_valid_ipv6(kwargs['ipv6_src_addr'])[-4:])
                 step = kwargs['ipv6_src_step']
-                if step < 1:
-                    raise STLError('ipv6_src_step has to be at least 1')
+
                 if type(step) is str: # convert ipv6 step to number
                     step = ipv4_str_to_num(is_valid_ipv6(step)[-4:])
+
+                if step < 1:
+                    raise STLError('ipv6_src_step has to be at least 1')
+                
                 if kwargs['ipv6_src_mode'] == 'increment':
                     add_val = ipv6_src_addr_num - 0x7fffffff
                     var_name = '%s_%s_%s_%s' % ('inc', 4, count, step)
@@ -1166,10 +1182,13 @@ def generate_packet(**user_kwargs):
             if count > 0 or kwargs['ipv6_dst_mode'] == 'random':
                 ipv6_dst_addr_num = ipv4_str_to_num(is_valid_ipv6(kwargs['ipv6_dst_addr'])[-4:])
                 step = kwargs['ipv6_dst_step']
-                if step < 1:
-                    raise STLError('ipv6_dst_step has to be at least 1')
+
                 if type(step) is str: # convert ipv6 step to number
                     step = ipv4_str_to_num(is_valid_ipv6(step)[-4:])
+
+                if step < 1:
+                    raise STLError('ipv6_dst_step has to be at least 1')
+                
                 if kwargs['ipv6_dst_mode'] == 'increment':
                     add_val = ipv6_dst_addr_num - 0x7fffffff
                     var_name = '%s_%s_%s_%s' % ('inc', 4, count, step)
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py
index f4a056f8..a7064853 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py
@@ -1136,7 +1136,7 @@ class STLPktBuilder(CTrexPktBuilderInterface):
              pkt : string, 
                 Scapy or pcap file filename a scapy packet 
 
-             pkt_buffer : string 
+             pkt_buffer : bytes 
                 a packet as buffer 
 
              vm   : list or base on :class:`trex_stl_lib.trex_stl_packet_builder_scapy.STLScVmRaw`
@@ -1156,6 +1156,9 @@ class STLPktBuilder(CTrexPktBuilderInterface):
         """
         super(STLPktBuilder, self).__init__()
 
+        validate_type('pkt', pkt, (type(None), str, Packet))
+        validate_type('pkt_buffer', pkt_buffer, (type(None), bytes))
+
         self.pkt = None     # as input
         self.pkt_raw = None # from raw pcap file
         self.vm_scripts = [] # list of high level instructions
@@ -1232,7 +1235,6 @@ class STLPktBuilder(CTrexPktBuilderInterface):
 
         """
         pkt_buf = self._get_pkt_as_str()
-
         return {'binary': base64.b64encode(pkt_buf).decode() if encode else pkt_buf,
                 'meta': self.metadata}
 
@@ -1266,7 +1268,7 @@ class STLPktBuilder(CTrexPktBuilderInterface):
 
 
     def set_pkt_as_str (self, pkt_buffer):
-        validate_type('pkt_buffer', pkt_buffer, str)
+        validate_type('pkt_buffer', pkt_buffer, bytes)
         self.pkt_raw = pkt_buffer
 
 
@@ -1288,7 +1290,7 @@ class STLPktBuilder(CTrexPktBuilderInterface):
 
         for pkt in p:
             was_set=True;
-            self.pkt_raw = str(pkt[0])
+            self.pkt_raw = pkt[0]
             break
         if not was_set :
             raise CTRexPacketBuildException(-14, "no buffer inside the pcap file {0}".format(f_path))
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
index e501c185..18678e3e 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_sim.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 """
@@ -22,6 +21,7 @@ from .trex_stl_exceptions import *
 from .trex_stl_streams import *
 from .utils import parsing_opts
 from .trex_stl_client import STLClient
+from .utils import pcap
 
 from yaml import YAMLError
 
@@ -33,45 +33,9 @@ import subprocess
 import os
 from operator import itemgetter
 
-# HACK
-import sys
-if sys.version_info < (3, 0):
-    from dpkt import pcap
-# HACK END
-
 class BpSimException(Exception):
     pass
 
-def merge_cap_files (pcap_file_list, out_filename, delete_src = False):
-
-    out_pkts = []
-    if not all([os.path.exists(f) for f in pcap_file_list]):
-        print("failed to merge cap file list...\nnot all files exist\n")
-        return
-
-    # read all packets to a list
-    for src in pcap_file_list:
-        f = open(src, 'r')
-        reader = pcap.Reader(f)
-        pkts = reader.readpkts()
-        out_pkts += pkts
-        f.close()
-        if delete_src:
-            os.unlink(src)
-
-    # sort by the timestamp
-    out_pkts = sorted(out_pkts, key=itemgetter(0))
-
-
-    out = open(out_filename, 'w')
-    out_writer = pcap.Writer(out)
-
-    for ts, pkt in out_pkts:
-        out_writer.writepkt(pkt, ts)
-
-    out.close()
-
-
 
 # stateless simulation
 class STLSim(object):
@@ -250,7 +214,7 @@ class STLSim(object):
         # write to temp file
         f = tempfile.NamedTemporaryFile(delete = False)
 
-        msg = json.dumps(cmds_json)
+        msg = json.dumps(cmds_json).encode()
 
         f.write(msg)
         f.close()
@@ -302,7 +266,7 @@ class STLSim(object):
         print("executing command: '{0}'".format(" ".join(cmd)))
 
         if self.silent:
-            FNULL = open(os.devnull, 'w')
+            FNULL = open(os.devnull, 'wb')
             rc = subprocess.call(cmd, stdout=FNULL)
         else:
             rc = subprocess.call(cmd)
@@ -325,8 +289,8 @@ class STLSim(object):
 
 
         print("Mering cores output to a single pcap file...\n")
-        inputs = ["{0}-{1}".format(self.outfile, index) for index in xrange(0, self.dp_core_count)]
-        merge_cap_files(inputs, self.outfile, delete_src = True)
+        inputs = ["{0}-{1}".format(self.outfile, index) for index in range(0, self.dp_core_count)]
+        pcap.merge_cap_files(inputs, self.outfile, delete_src = True)
 
 
 
@@ -365,7 +329,7 @@ def setParserOptions():
                         dest = "dp_core_count",
                         default = 1,
                         type = int,
-                        choices = xrange(1, 9))
+                        choices = list(range(1, 9)))
 
     parser.add_argument("-n", "--core_index",
                         help = "Record only a specific core",
@@ -460,7 +424,7 @@ def setParserOptions():
 def validate_args (parser, options):
 
     if options.dp_core_index:
-        if not options.dp_core_index in xrange(0, options.dp_core_count):
+        if not options.dp_core_index in range(0, options.dp_core_count):
             parser.error("DP core index valid range is 0 to {0}".format(options.dp_core_count - 1))
 
     # zero is ok - no limit, but other values must be at least as the number of cores
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
index c258c749..16ffad43 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_streams.py
@@ -525,8 +525,15 @@ class STLStream(object):
         if payload:
             payload.remove_payload() # fcs etc.
             data = payload.fields.get('load', '')
-            replchars = re.compile('(\s|/|\'|\\\|[^' + re.escape(string.printable) + '])') # convert bad chars to hex
-            new_data =  replchars.sub(self.__replchars_to_hex, data)
+
+            good_printable = [c for c in string.printable if ord(c) not in range(32)]
+            good_printable.remove("'")
+
+            if type(data) is str:
+                new_data = ''.join([c if c in good_printable else r'\x{0:02x}'.format(ord(c)) for c in x])
+            else:
+                new_data = ''.join([chr(c) if chr(c) in good_printable else r'\x{0:02x}'.format(c) for c in x])
+
             payload_start = packet_command.find("Raw(load='")
             if payload_start != -1:
                 packet_command = packet_command[:payload_start-1]
@@ -931,7 +938,7 @@ class STLProfile(object):
             raise STLError("file '{0}' does not exists".format(pcap_file))
 
         # make sure IPG is not less than 1 usec
-        if ipg_usec < 1:
+        if ipg_usec is not None and ipg_usec < 1:
             raise STLError("ipg_usec cannot be less than 1 usec: '{0}'".format(ipg_usec))
 
         if loop_count < 0:
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
index 6ec2d189..c4f2b358 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/parsing_opts.py
@@ -225,7 +225,7 @@ OPTIONS_DB = {MULTIPLIER: ArgumentPack(['-m', '--multiplier'],
               IPG: ArgumentPack(['-i', '--ipg'],
                                 {'help': "IPG value in usec between packets. default will be from the pcap",
                                  'dest': "ipg_usec",
-                                 'default':  100.0,
+                                 'default':  None,
                                  'type': float}),
 
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/pcap.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/pcap.py
new file mode 100644
index 00000000..ab4f98a7
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/utils/pcap.py
@@ -0,0 +1,29 @@
+import os
+from ..trex_stl_packet_builder_scapy import RawPcapReader, RawPcapWriter
+
+
+def __ts_key (a):
+    return float(a[1][0]) + (float(a[1][1]) / 1e6)
+
+def merge_cap_files (pcap_file_list, out_filename, delete_src = False):
+
+    if not all([os.path.exists(f) for f in pcap_file_list]):
+        print("failed to merge cap file list...\nnot all files exist\n")
+        return
+
+    out_pkts = []
+    for src in pcap_file_list:
+        pkts = RawPcapReader(src)
+        out_pkts += pkts
+        if delete_src:
+            os.unlink(src)
+
+    # sort by timestamp
+    out_pkts = sorted(out_pkts, key = __ts_key)
+
+    writer = RawPcapWriter(out_filename, linktype = 1)
+
+    writer._write_header(None)
+    for pkt in out_pkts:
+        writer._write_packet(pkt[0], sec=pkt[1][0], usec=pkt[1][1], caplen=pkt[1][2], wirelen=None)
+