scapy_server_test:

added cases for testing: GRE, VXML, DNS, IPv6
changed test functions to include only the testted object and created other  functions as testing engines.

scapy_service:
added api class.
added documentry for api class
changed method functions to include underscore

3 files changed, 204 insertions, 142 deletions

diff --git a/scripts/automation/regression/functional_tests/scapy_server_test.py b/scripts/automation/regression/functional_tests/scapy_server_test.py
index ccf0b754..c0651f69 100755
--- a/scripts/automation/regression/functional_tests/scapy_server_test.py
+++ b/scripts/automation/regression/functional_tests/scapy_server_test.py
@@ -27,51 +27,7 @@ import zmq
 import json

 import scapy_zmq_server

 import threading

-

-

-class Scapy_server_wrapper():

-    def __init__(self,dest_scapy_port=5555,server_ip_address='localhost'):

-        self.context = zmq.Context()

-        self.socket = self.context.socket(zmq.REQ)

-        self.dest_scapy_port =dest_scapy_port

-        self.socket.connect("tcp://"+str(server_ip_address)+":"+str(self.dest_scapy_port)) #ip address of csi-trex-11

-

-    def call_method(self,method_name,method_params):

-        json_rpc_req = { "jsonrpc":"2.0","method": method_name ,"params": method_params, "id":"1"}

-        request = json.dumps(json_rpc_req)

-        self.socket.send(request)

-        #  Get the reply.

-        message = self.socket.recv()

-#       print("Received reply %s [ %s ]" % (request, message))

-        message_parsed = json.loads(message)

-        try:

-            result = message_parsed['result']

-        except:

-            result = {'error':message_parsed['error']}

-        finally:

-            return result

-

-    def get_all(self):

-        return self.call_method('get_all',[])

-

-    def check_update(self,db_md5,field_md5):

-        result = self.call_method('check_update',[db_md5,field_md5])

-        if result!=True:

-            if 'error' in result.keys():

-                if "Fields DB is not up to date" in result['error']['message:']:

-                    raise ScapyException("Fields DB is not up to date")

-                if "Protocol DB is not up to date" in result['error']['message:']:

-                    raise ScapyException("Protocol DB is not up to date")

-        return result

-

-    def build_pkt(self,pkt_descriptor):

-        return self.call_method('build_pkt',[pkt_descriptor])

-        

-    def get_all_pkt_offsets(self,pkt_desc):

-        return self.call_method('get_all_pkt_offsets',[pkt_desc])

-

-

-

+from scapy_zmq_client import Scapy_server_wrapper

 

 class scapy_service_tester(functional_general_test.CGeneralFunctional_Test):

     def setUp(self):

@@ -133,8 +89,7 @@ class scapy_service_tester(functional_general_test.CGeneralFunctional_Test):
             raise Exception("scapy_server_test: check_updating_db failed")

 

 

-# testing pkt = Ether()/IP()/TCP()/"test" by defualt

-    def test_build_packet(self,original_pkt='Ether()/IP()/TCP()/"test"'):

+    def _build_packet_test_method(self,original_pkt):

         test_pkt = original_pkt

         original_pkt = eval(original_pkt)

         test_res = self.s.build_pkt(test_pkt)

@@ -143,12 +98,12 @@ class scapy_service_tester(functional_general_test.CGeneralFunctional_Test):
         assert_equal(resT1,True)

 

 

-#testing offsets of packet IP() by default

-    def test_get_all_offsets(self,original_pkt = 'IP()'):

+#testing offsets of a packet

+    def _get_all_offsets_test_method(self,original_pkt):

         test_pkt = original_pkt

         original_pkt = eval(original_pkt)

         original_pkt.build()

-        tested_offsets_by_layers = self.s.get_all_pkt_offsets(test_pkt)

+        tested_offsets_by_layers = self.s._get_all_pkt_offsets(test_pkt)

         layers = (test_pkt).split('/')

         offsets_by_layers = {}

         for layer in layers:

@@ -167,25 +122,11 @@ class scapy_service_tester(functional_general_test.CGeneralFunctional_Test):
             offsets_by_layers[layer_name] = fields_dict

         resT1 = (tested_offsets_by_layers == offsets_by_layers)

         assert_equal(resT1,True)

-

-    def test_multi_packet(self):

-        e0 = 'Ether()'

-        e1 = 'Ether()/IP()'

-        e2 = 'TCP()'

-        e3 = 'UDP()'

-        e4 = 'Ether()/IP()/TCP()/"test"'

-        e5 = 'Ether()/IP()/UDP()'

-        packets = [e0,e1,e2,e3,e4,e5]

-        for packet in packets:

-            self.test_get_all_offsets(packet)

-

-        for packet in packets:

-            self.test_build_packet(packet)

-

-    def test_offsets_and_buffer(self,mac_src='ab:cd:ef:12:34:56',mac_dst='98:76:54:32:1a:bc',ip_src='127.1.1.1',ip_dst='192.168.1.1'):

+    

+    def _offsets_and_buffer_test_method(self,mac_src,mac_dst,ip_src,ip_dst):

         pkt = Ether(src=mac_src,dst=mac_dst)/IP(src=ip_src,dst=ip_dst)/TCP()

         pkt_descriptor = "Ether(src='"+mac_src+"',dst='"+mac_dst+"')/IP(src='"+ip_src+"',dst='"+ip_dst+"')/TCP()"

-        pkt_offsets = self.s.get_all_pkt_offsets(pkt_descriptor)

+        pkt_offsets = self.s._get_all_pkt_offsets(pkt_descriptor)

         pkt_buffer = str(pkt)

         #--------------------------Dest-MAC--------------------

         mac_start_index = pkt_offsets['Ether']['dst'][0]+pkt_offsets['Ether']['global_offset']

@@ -204,7 +145,52 @@ class scapy_service_tester(functional_general_test.CGeneralFunctional_Test):
         ip_end_index= ip_start_index+pkt_offsets['IP']['src'][1]

         assert_equal(binascii.b2a_hex(pkt_buffer[ip_start_index:ip_end_index]),binascii.hexlify(socket.inet_aton(ip_src)))

 

+    def test_multi_packet(self):

+        packets= [

+            'Ether()',

+            'Ether()/IP()',

+            'TCP()',

+            'UDP()',

+            'Ether()/IP()/TCP()/"test"',

+            'Ether()/IP()/UDP()',

+            'Ether()/IP(src="16.0.0.1",dst="48.0.0.1")',

+            'Ether()/IP(src="16.0.0.1",dst="48.0.0.1")',

+            'Ether()/Dot1Q(vlan=12)/Dot1Q(vlan=12)/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)',

+            'Ether()/Dot1Q(vlan=12)/IP(src="16.0.0.1",dst="48.0.0.1")/TCP(dport=12,sport=1025)',

+            'Ether()/Dot1Q(vlan=12)/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)',

+            'Ether()/Dot1Q(vlan=12)/IPv6(src="::5")/TCP(dport=12,sport=1025)',

+            'Ether()/IP()/UDP()/IPv6(src="::5")/TCP(dport=12,sport=1025)',

+            'Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)',

+            'Ether()/IP(dst="48.0.0.1")/TCP(dport=80,flags="S")',

+            'Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)',

+            'Ether() / IP(src = "16.0.0.1", dst = "48.0.0.1") / UDP(dport = 12, sport = 1025)',

+            'Ether()/IP()/UDP()',

+            'Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)',

+            'Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)',

+            'Ether()/IP(src="16.0.0.2",dst="48.0.0.1")/UDP(dport=12,sport=1025)',

+            'Ether()/IP(src="16.0.0.3",dst="48.0.0.1")/UDP(dport=12,sport=1025)',

+            r'Ether()/IP()/IPv6()/IP(dst="48.0.0.1",options=IPOption("\x01\x01\x01\x00"))/UDP(dport=12,sport=1025)',

+            r'Ether()/IP(dst="48.0.0.1",options=IPOption("\x01\x01\x01\x00"))/UDP(dport=12,sport=1025)',

+            'Ether()',

+            'Ether()/IP()/UDP(sport=1337,dport=4789)/VXLAN(vni=42)/Ether()/IP()/("x"*20)',

+            'Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=3797,sport=3544)/IPv6(dst="2001:0:4137:9350:8000:f12a:b9c8:2815",src="2001:4860:0:2001::68")/UDP(dport=12,sport=1025)/ICMPv6Unknown()',

+            'Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(sport=1025)/DNS()',

+            'Ether()/MPLS(label=17,cos=1,s=0,ttl=255)/MPLS(label=0,cos=1,s=1,ttl=12)/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/("x"*20)',

+            'Ether()/MPLS(label=17,cos=1,s=0,ttl=255)/MPLS(label=12,cos=1,s=1,ttl=12)/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/("x"*20)',

+            'Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/ICMP(type=3)',

+            'Ether()/IP()/GRE()/("x"*2)']

+

+        for packet in packets:

+            self._get_all_offsets_test_method(packet)

+

+        for packet in packets:

+            self._build_packet_test_method(packet)

+

+    

 

+    def test_offsets_and_buffer(self):

+        self._offsets_and_buffer_test_method('ab:cd:ef:12:34:56','98:76:54:32:1a:bc','127.1.1.1','192.168.1.1')

+        self._offsets_and_buffer_test_method('bb:bb:bb:bb:bb:bb','aa:aa:aa:aa:aa:aa','1.1.1.1','0.0.0.0')

 

 class scapy_server_thread(threading.Thread):

     def __init__(self,thread_id,server_port=5555):

@@ -213,9 +199,9 @@ class scapy_server_thread(threading.Thread):
         self.server_port = server_port

 

     def run(self):

-        print '\nStarted scapy thread server'

+        print('\nStarted scapy thread server')

         scapy_zmq_server.main(self.server_port)

-        print 'Thread server closed'

+        print('Thread server closed')

 

 # Scapy_server_wrapper is the CLIENT for the scapy server, it wraps the CLIENT: its default port is set to 5555, default server ip set to localhost

 class scapy_server_tester(scapy_service_tester):

diff --git a/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_service.py b/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_service.py
index 638d35f2..3519bb58 100755
--- a/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_service.py
+++ b/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_service.py
@@ -2,7 +2,9 @@
 import os
 import sys
 stl_pathname = os.path.abspath(os.path.join(os.pardir, os.pardir))
+additional_stl_udp_pkts = os.path.abspath(os.path.join(os.pardir, os.pardir,'stl'))
 sys.path.append(stl_pathname)
+sys.path.append(additional_stl_udp_pkts)
 import trex_stl_lib
 from trex_stl_lib.api import *
 from copy import deepcopy
@@ -11,32 +13,106 @@ import tempfile
 import hashlib
 import binascii
 from pprint import pprint
-
+from scapy.layers.dns import *
+from udp_1pkt_vxlan import VXLAN
+from udp_1pkt_mpls import MPLS
 
 try:
     from cStringIO import StringIO
 except ImportError:
     from io import StringIO
 
-"""     
-                            **** output redirection template ****
-old_stdout = sys.stdout
-sys.stdout = mystdout = StringIO()
 
-ls()
 
-sys.stdout = old_stdout
 
-a= mystdout.getvalue()
+class Scapy_service_api():
+    """ get_all(self) 
 
-f = open('scapy_supported_formats.txt','w')
-f.write(a)
-f.close()
-"""
+        Sends all the protocols and fields that Scapy Service supports.
+        also sends the md5 of the Protocol DB and Fields DB used to check if the DB's are up to date
 
-class ScapyException(Exception): pass
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        Dictionary (of protocol DB and scapy fields DB)
+
+        Raises
+        ------
+        Raises an exception when a DB error occurs (i.e a layer is not loaded properly and has missing components)
+    """
+    def get_all(self):
+        pass
+
+    """ check_update(self,db_md5,field_md5) 
+        Checks if the Scapy Service running on the server has a newer version of the databases that the client has
+
+        Parameters
+        ----------
+        db_md5 - The md5 that was delivered with the protocol database that the client owns, when first received at the client
+        field_md5 - The md5 that was delivered with the fields database that the client owns, when first received at the client
+
+        Returns
+        -------
+        True/False according the Databases version(determined by their md5)
+
+        Raises
+        ------
+        Raises an exception (ScapyException) when protocol DB/Fields DB is not up to date
+
+    """
 
-class Scapy_service:
+    def check_update(self,db_md5,field_md5):        
+        pass
+
+    """ build_pkt(self,pkt_descriptor) -> Dictionary (of Offsets,Show2 and Buffer)
+        
+        Performs calculations on the given packet and returns results for that packet.
+    
+        Parameters
+        ----------
+        pkt_descriptor - A string describing a network packet, in Scapy Format
+
+        Returns
+        -------
+        - The packets offsets: each field in every layer is mapped inside the Offsets Dictionary
+        - The Show2: A description of each field and its value in every layer of the packet
+        - The Buffer: The Hexdump of packet encoded in base64
+
+        Raises
+        ------
+        will raise an exception when the Scapy string format is illegal, contains syntax error, contains non-supported
+        protocl, etc.
+    """
+    def build_pkt(self,pkt_descriptor):
+        pass
+
+    """ get_tree(self) -> Dictionary describing an example of hierarchy in layers
+
+        Scapy service holds a tree of layers that can be stacked to a recommended packet
+        according to the hierarchy
+
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        Returns an example hierarchy tree of layers that can be stacked to a packet
+
+        Raises
+        ------
+        None
+    """
+    def get_tree(self):
+        pass
+
+
+
+class ScapyException(Exception): pass
+class Scapy_service(Scapy_service_api):
 
 #----------------------------------------------------------------------------------------------------
     class scapyRegex:
@@ -55,10 +131,10 @@ class Scapy_service:
         self.low_level_protocols = { 'Ether': self.network_protocols }
         self.regexDB= {'MACField' : self.scapyRegex('MACField','^([0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])$'),
               'IPField' : self.scapyRegex('IPField','^(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])$')}
-        self.all_protocols = self.build_lib()
+        self.all_protocols = self._build_lib()
         self.protocol_tree = {'ALL':{'Ether':{'ARP':{},'IP':{'TCP':{'RAW':'payload'},'UDP':{'RAW':'payload'}}}}}
     
-    def protocol_struct(self,protocol=''):
+    def _protocol_struct(self,protocol=''):
         if '_' in protocol:
             return []
         if not protocol=='':
@@ -75,8 +151,8 @@ class Scapy_service:
         protocol_data= mystdout.getvalue()
         return protocol_data
 
-    def build_lib(self):
-        lib = self.protocol_struct()
+    def _build_lib(self):
+        lib = self._protocol_struct()
         lib = lib.split('\n')
         all_protocols=[]
         for entry in lib:
@@ -85,45 +161,39 @@ class Scapy_service:
         del all_protocols[len(all_protocols)-1]
         return all_protocols
 
-    def parse_description_line(self,line):
+    def _parse_description_line(self,line):
         line_arr = [x.strip() for x in re.split(': | = ',line)]
         return tuple(line_arr)
 
-    def parse_entire_description(self,description):
+    def _parse_entire_description(self,description):
         description = description.split('\n')
-        description_list = [self.parse_description_line(x) for x in description]
+        description_list = [self._parse_description_line(x) for x in description]
         del description_list[len(description_list)-1]
         return description_list
 
-    def get_protocol_details(self,p_name):
-        protocol_str = self.protocol_struct(p_name)
+    def _get_protocol_details(self,p_name):
+        protocol_str = self._protocol_struct(p_name)
         if protocol_str=='protocol not supported':
             return 'protocol not supported'
         if len(protocol_str) is 0:
             return []
-        tupled_protocol = self.parse_entire_description(protocol_str)
+        tupled_protocol = self._parse_entire_description(protocol_str)
         return tupled_protocol
 
-    def print_tree(self):
+    def _print_tree(self):
         pprint(self.protocol_tree)
 
-    def get_all_protocols(self):
-        return self.all_protocols
-
-    def get_tree(self):
-        return self.protocol_tree
-
-    def get_all_db(self):
+    def _get_all_db(self):
         db = {}
         for pro in self.all_protocols:
-            details = self.get_protocol_details(pro)
+            details = self._get_protocol_details(pro)
             db[pro] = details
         return db
 
-    def get_all_fields(self):
+    def _get_all_fields(self):
         fields = []
         for pro in self.all_protocols:
-            details = self.get_protocol_details(pro)
+            details = self._get_protocol_details(pro)
             for i in range(0,len(details),1):
                 if len(details[i]) is 3:
                     fields.append(details[i][1])
@@ -136,7 +206,7 @@ class Scapy_service:
                 fieldDict[f] = self.scapyRegex(f).stringRegex()
         return fieldDict
 
-    def show2_to_dict(self,pkt):
+    def _show2_to_dict(self,pkt):
         old_stdout = sys.stdout
         sys.stdout = mystdout = StringIO()
         pkt.show2()
@@ -160,7 +230,7 @@ class Scapy_service:
 
 #pkt_desc as string
 #dictionary of offsets per protocol. tuple for each field: (name, offset, size) at json format
-    def get_all_pkt_offsets(self,pkt_desc):
+    def _get_all_pkt_offsets(self,pkt_desc):
         pkt_protocols = pkt_desc.split('/')
         scapy_pkt = eval(pkt_desc)
         scapy_pkt.build()
@@ -181,34 +251,51 @@ class Scapy_service:
             scapy_pkt=scapy_pkt.payload
         return res
 
-# pkt_descriptor in string format
+#input: container
+#output: md5 encoded in base64
+    def _get_md5(self,container):
+        container = json.dumps(container)
+        m = hashlib.md5()
+        m.update(container.encode('ascii'))
+        res_md5 = binascii.b2a_base64(m.digest())
+        return res_md5
 
+    def get_version(self):
+        return {'built_by':'itraviv','version':'v1.0'}
+
+    def supported_methods(self,method_name=''):
+        if method_name=='':
+            methods = {}
+            for f in dir(Scapy_service):
+                if inspect.ismethod(eval('Scapy_service.'+f)):
+                    methods[f] = inspect.getargspec(eval('Scapy_service.'+f))[0]
+            return methods
+        if method_name in dir(Scapy_service):
+            return True
+        return False
+
+#--------------------------------------------API implementation-------------
+    def get_tree(self):
+        return self.protocol_tree
+
+# pkt_descriptor in string format
     def build_pkt(self,pkt_descriptor):
         pkt = eval(pkt_descriptor)
-        show2data = self.show2_to_dict(pkt)
+        show2data = self._show2_to_dict(pkt)
         bufferData = str(pkt) #pkt buffer
         bufferData = binascii.b2a_base64(bufferData)
-        pkt_offsets = self.get_all_pkt_offsets(pkt_descriptor)
+        pkt_offsets = self._get_all_pkt_offsets(pkt_descriptor)
         res = {}
         res['show2'] = show2data
         res['buffer'] = bufferData
         res['offsets'] = pkt_offsets
         return res
 
-#input: container
-#output: md5 encoded in base64
-    def get_md5(self,container):
-        container = json.dumps(container)
-        m = hashlib.md5()
-        m.update(container.encode('ascii'))
-        res_md5 = binascii.b2a_base64(m.digest())
-        return res_md5
-
     def get_all(self):
-        fields=self.get_all_fields()
-        db=self.get_all_db()
-        fields_md5 = self.get_md5(fields)
-        db_md5 = self.get_md5(db)
+        fields=self._get_all_fields()
+        db=self._get_all_db()
+        fields_md5 = self._get_md5(fields)
+        db_md5 = self._get_md5(db)
         res = {}
         res['db'] = db
         res['fields'] = fields
@@ -218,10 +305,10 @@ class Scapy_service:
 
 #input in string encoded base64
     def check_update(self,db_md5,field_md5):
-        fields=self.get_all_fields()
-        db=self.get_all_db()
-        current_db_md5 = self.get_md5(db)
-        current_field_md5 = self.get_md5(fields)
+        fields=self._get_all_fields()
+        db=self._get_all_db()
+        current_db_md5 = self._get_md5(db)
+        current_field_md5 = self._get_md5(fields)
         res = []
         if (field_md5.decode("base64") == current_field_md5.decode("base64")):
             if (db_md5.decode("base64") == current_db_md5.decode("base64")):
@@ -231,18 +318,7 @@ class Scapy_service:
         else:
             raise ScapyException("Fields DB is not up to date")
 
-    def get_version(self):
-        return {'built_by':'itraviv','version':'v1.0'}
 
-    def supported_methods(self,method_name=''):
-        if method_name=='':
-            methods = {}
-            for f in dir(Scapy_service):
-                if inspect.ismethod(eval('Scapy_service.'+f)):
-                    methods[f] = inspect.getargspec(eval('Scapy_service.'+f))[0]
-            return methods
-        if method_name in dir(Scapy_service):
-            return True
-        return False
+#---------------------------------------------------------------------------
 
 
diff --git a/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_zmq_server.py b/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_zmq_server.py
index b2f2d526..408f1daf 100755
--- a/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_zmq_server.py
+++ b/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_zmq_server.py
@@ -106,16 +106,16 @@ class Scapy_server():
         self.IP_address = socket.gethostbyname(socket.gethostname())

 

     def activate(self):

-        print '***Scapy Server Started***\nListening on port: %d' % self.port

-        print 'Server IP address: %s' % self.IP_address

+        print ('***Scapy Server Started***\nListening on port: %d' % self.port)

+        print ('Server IP address: %s' % self.IP_address)

         try:

             while True:

                 message = self.socket.recv()

                 try:

                     method,params,req_id = self.scapy_wrapper.parse_req_msg(message)

                     if (method == 'shut_down'):

-                        print 'Shut down by remote user'

-                        result = 'Server shut down command received - server has shut down'

+                        print ('Shut down by remote user')

+                        result = 'Server shut down command received - server had shut down'

                     else:

                         result = self.scapy_wrapper.execute(method,params)

                     response = self.scapy_wrapper.create_success_response(result,req_id)