add protocol definition json for scapy server

Signed-off-by: Anton Kiselev <anton.kisel@gmail.com>

4 files changed, 254 insertions, 9 deletions

diff --git a/scripts/automation/trex_control_plane/stl/services/scapy_server/protocols.json b/scripts/automation/trex_control_plane/stl/services/scapy_server/protocols.json
new file mode 100644
index 00000000..f685c06f
--- /dev/null
+++ b/scripts/automation/trex_control_plane/stl/services/scapy_server/protocols.json
@@ -0,0 +1,194 @@
+[
+  {
+    "id": "Ether",
+    "name": "Ethernet II",
+    "fields": [
+      {
+        "id": "dst",
+        "name": "Destination",
+        "type": "MAC_ADDRESS",
+        "regex": "^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$"
+      },
+      {
+        "id": "src",
+        "name": "Source",
+        "type": "MAC_ADDRESS",
+        "regex": "^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$"
+      },
+      {
+        "id": "type",
+        "name": "Type"
+      }
+    ],
+    "payload": ["IP", "IPv6", "Dot1Q", "Raw"]
+  },
+  {
+    "id": "IP",
+    "name": "IPv4",
+    "fields": [
+      {
+        "id": "version",
+        "name": "Version"
+      },
+      {
+        "id": "ihl",
+        "name": "IHL",
+        "type": "NUMBER",
+        "auto": true
+      },
+      {
+        "id": "tos",
+        "name": "TOS",
+        "type": "NUMBER"
+      },
+      {
+        "id": "len",
+        "name": "Total Length",
+        "type": "NUMBER",
+        "auto": true
+      },
+      {
+        "id": "id",
+        "name": "Identification",
+        "type": "NUMBER"
+      },
+      {
+        "id": "flags",
+        "name": "Flags",
+        "type": "BITMASK",
+        "bits": [
+          {"name": "Reserved", "mask": 4, "values":[{"name":"Not Set", "value": 0}, {"name":"Set", "value": 4}]},
+          {"name": "Fragment", "mask": 2, "values":[{"name":"May fragment (0)", "value": 0}, {"name":"Don't fragment (1)", "value": 2}]},
+          {"name": "More Fragments(MF)", "mask": 1, "values":[{"name":"Not Set", "value": 0}, {"name":"Set", "value": 1}]}
+        ]
+      },
+      {
+        "id": "frag",
+        "name": "Fragment offset",
+        "type": "NUMBER"
+      },
+      {
+        "id": "ttl",
+        "name": "TTL",
+        "type": "NUMBER",
+        "min": 1,
+        "max": 255
+        
+      },
+      {
+        "id": "proto",
+        "name": "Protocol"
+      },
+      {
+        "id": "chksum",
+        "name": "Checksum",
+        "type": "STRING",
+        "auto": true
+      },
+      {
+        "id": "src",
+        "name": "Source address",
+        "type": "IP_ADDRESS"
+      },
+      {
+        "id": "dst",
+        "name": "Destination address",
+        "type": "IP_ADDRESS"
+      },
+      {
+        "id": "options",
+        "name": "Options",
+        "type": "IP_OPTIONS"
+      }
+    ],
+    "payload": ["TCP", "UDP", "ICMP", "Raw"]
+  },
+  {
+    "id": "TCP",
+    "name": "TCP",
+    "fields": [
+      {
+        "id": "sport",
+        "name": "Source port",
+        "type": "NUMBER",
+        "min": 0,
+        "max": 65535
+        
+      },
+      {
+        "id": "dport",
+        "name": "Destination port",
+        "type": "NUMBER",
+        "min": 0,
+        "max": 65535
+      },
+      {
+        "id": "seq",
+        "name": "Sequence number",
+        "type": "NUMBER"
+      },
+      {
+        "id": "ack",
+        "name": "Acknowledgment number",
+        "type": "NUMBER"
+      },
+      {
+        "id": "dataofs",
+        "name": "Data offset",
+        "type": "NUMBER"
+      },
+      {
+        "id": "reserved",
+        "name": "Reserved",
+        "type": "NUMBER"
+      },
+      {
+        "id": "flags",
+        "name": "Flags",
+        "auto": false,
+        "type": "BITMASK",
+        "bits": [
+          {"name": "URG", "mask": 32, "values":[{"name":"Not Set", "value": 0}, {"name":"Set", "value": 32}]},
+          {"name": "ACK", "mask": 16, "values":[{"name":"Not Set", "value": 0}, {"name":"Set", "value": 16}]},
+          {"name": "PSH", "mask": 8, "values":[{"name":"Not Set", "value": 0}, {"name":"Set", "value": 8}]},
+          {"name": "RST", "mask": 4, "values":[{"name":"Not Set", "value": 0}, {"name":"Set", "value": 4}]},
+          {"name": "SYN", "mask": 2, "values":[{"name":"Not Set", "value": 0}, {"name":"Set", "value": 2}]},
+          {"name": "FIN", "mask": 1, "values":[{"name":"Not Set", "value": 0}, {"name":"Set", "value": 1}]}
+        ]
+      },
+      {
+        "id": "window",
+        "name": "Window size",
+        "type": "NUMBER"
+      },
+      {
+        "id": "chksum",
+        "name": "Checksum",
+        "auto": true,
+        "type": "NUMBER"
+      },
+      {
+        "id": "urgptr",
+        "name": "Urgent pointer",
+        "type": "NUMBER"
+      },
+      {
+        "id": "options",
+        "name": "Options",
+        "type": "TCP_OPTIONS"
+      }
+    ]
+  },
+  {
+    "id": "Raw",
+    "name": "Raw",
+    "fields": [
+      {
+        "id": "load",
+        "name": "Payload",
+        "type": "BYTES"
+      }
+    ]
+  }
+]
+
diff --git a/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_service.py b/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_service.py
index 91257596..4ce9e4ae 100755
--- a/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_service.py
+++ b/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_service.py
@@ -312,7 +312,16 @@ class Scapy_service(Scapy_service_api):
         self.version_major = '1'
         self.version_minor = '01'
         self.server_v_hashed = self._generate_version_hash(self.version_major,self.version_minor)
-    
+        self.protocol_definitions = {} # protocolId -> prococol definition overrides data
+        self._load_definitions_from_json()
+
+    def _load_definitions_from_json(self):
+        # load protocol definitions from a json file
+        self.protocol_definitions = {}
+        with file('protocols.json') as f:
+            protocols = json.load(f)
+            for protocol in protocols:
+                self.protocol_definitions[ protocol['id'] ] = protocol
 
     def _all_protocol_structs(self):
         old_stdout = sys.stdout
@@ -654,10 +663,9 @@ class Scapy_service(Scapy_service_api):
             return pkt_class()
 
         
-    def _get_payload_classes(self, pkt):
+    def _get_payload_classes(self, pkt_class):
         # tries to find, which subclasses allowed.
         # this can take long time, since it tries to build packets with all subclasses(O(N))
-        pkt_class = type(pkt)
         allowed_subclasses = []
         for pkt_subclass in conf.layers:
             if self._is_packet_class(pkt_subclass):
@@ -671,16 +679,29 @@ class Scapy_service(Scapy_service_api):
                     pass
         return allowed_subclasses
 
-    def _get_fields_definition(self, pkt_class):
+    def _get_fields_definition(self, pkt_class, fieldsDef):
+        # fieldsDef - array of field definitions(or empty array)
         fields = []
         for field_desc in pkt_class.fields_desc:
+            fieldId = field_desc.name
             field_data = {
-                    "id": field_desc.name,
+                    "id": fieldId,
                     "name": field_desc.name
             }
+            for fieldDef in fieldsDef:
+                if fieldDef['id'] == fieldId:
+                    field_data.update(fieldDef)
             if isinstance(field_desc, EnumField):
                 try:
                     field_data["values_dict"] = field_desc.s2i
+                    if field_data.get("type") == None:
+                        if len(field_data["values_dict"] > 0):
+                            field_data["type"] = "ENUM"
+                        elif fieldId == 'load':
+                            field_data["type"] = "BYTES"
+                        else:
+                            field_data["type"] = "STRING"
+                    field_data["values_dict"] = field_desc.s2i
                 except:
                     # MultiEnumField doesn't have s2i. need better handling
                     pass
@@ -696,17 +717,23 @@ class Scapy_service(Scapy_service_api):
         for pkt_class in all_classes:
             if self._is_packet_class(pkt_class):
                 # enumerate all non-abstract Packet classes
+                protocolId = pkt_class.__name__
+                protoDef = self.protocol_definitions.get(protocolId) or {}
                 protocols.append({
-                    "id": pkt_class.__name__,
-                    "name": pkt_class.name,
-                    "fields": self._get_fields_definition(pkt_class)
+                    "id": protocolId,
+                    "name": protoDef.get('name') or pkt_class.name,
+                    "fields": self._get_fields_definition(pkt_class, protoDef.get('fields') or [])
                     })
         res = {"protocols": protocols}
         return res
 
     def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
         pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
-        return [c.__name__ for c in self._get_payload_classes(pkt)]
+        pkt_class = type(pkt.lastlayer())
+        protocolDef = self.protocol_definitions.get(pkt_class.__name__)
+        if protocolDef and protocolDef.get('payload'):
+            return protocolDef['payload']
+        return [c.__name__ for c in self._get_payload_classes(pkt_class)]
 
 #input in string encoded base64
     def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
diff --git a/scripts/automation/trex_control_plane/stl/services/scapy_server/unit_tests/basetest.py b/scripts/automation/trex_control_plane/stl/services/scapy_server/unit_tests/basetest.py
index 17dd304a..1db2c62b 100644
--- a/scripts/automation/trex_control_plane/stl/services/scapy_server/unit_tests/basetest.py
+++ b/scripts/automation/trex_control_plane/stl/services/scapy_server/unit_tests/basetest.py
@@ -62,6 +62,9 @@ def reconstruct_pkt(bytes_b64, model_def):
 def get_definitions(def_filter):
     return pass_result(service.get_definitions(v_handler, def_filter))
 
+def get_definition_of(scapy_classname):
+    return pass_result(service.get_definitions(v_handler, [scapy_classname]))['protocols'][0]
+
 def get_payload_classes(def_filter):
     return pass_result(service.get_payload_classes(v_handler, def_filter))
 
diff --git a/scripts/automation/trex_control_plane/stl/services/scapy_server/unit_tests/test_scapy_service.py b/scripts/automation/trex_control_plane/stl/services/scapy_server/unit_tests/test_scapy_service.py
index 9cd473d7..7126ef4b 100644
--- a/scripts/automation/trex_control_plane/stl/services/scapy_server/unit_tests/test_scapy_service.py
+++ b/scripts/automation/trex_control_plane/stl/services/scapy_server/unit_tests/test_scapy_service.py
@@ -98,6 +98,16 @@ def test_get_payload_classes():
     assert("IP" in eth_payloads)
     assert("Dot1Q" in eth_payloads)
     assert("TCP" not in eth_payloads)
+    assert(eth_payloads[0] == "IP") # order(based on prococols.json)
+
+def test_get_tcp_payload_classes():
+    payloads = get_payload_classes([{"id":"TCP"}])
+    assert("Raw" in payloads)
+
+def test_get_dot1q_payload_classes():
+    payloads = get_payload_classes([{"id":"Dot1Q"}])
+    assert("Dot1Q" in payloads)
+    assert("IP" in payloads)
 
 def test_pcap_read_and_write():
     pkts_to_write = [bytes_to_b64(bytes(TEST_PKT))]
@@ -153,3 +163,14 @@ def test_layer_wrong_structure():
     assert(real_structure == ["Ether", "IP", "Raw", None, None])
     assert(valid_structure_flags == [True, True, True, False, False])
 
+def test_hand_crafted_definitions():
+    etherDef = get_definition_of("Ether")
+    assert(etherDef['name'] == "Ethernet II")
+    etherFields = etherDef['fields']
+    assert(etherFields[0]['id'] == 'dst')
+    assert(etherFields[0]['name'] == 'Destination')
+    assert(etherFields[1]['id'] == 'src')
+    assert(etherFields[1]['name'] == 'Source')
+    assert(etherFields[2]['id'] == 'type')
+    assert(etherFields[2]['name'] == 'Type')
+