1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
|
import os
import sys
stl_pathname = os.path.abspath(os.path.join(os.pardir, os.pardir))
sys.path.append(stl_pathname)
from trex_stl_lib.api import *
import tempfile
import hashlib
import base64
import numbers
import random
import inspect
import json
from pprint import pprint
# add some layers as an example
# need to test more
from scapy.layers.dns import *
from scapy.layers.dhcp import *
from scapy.layers.ipsec import *
from scapy.layers.netflow import *
from scapy.layers.sctp import *
from scapy.layers.tftp import *
from scapy.contrib.mpls import *
from scapy.contrib.igmp import *
from scapy.contrib.igmpv3 import *
#additional_stl_udp_pkts = os.path.abspath(os.path.join(os.pardir,os.pardir,os.pardir,os.pardir, os.pardir,'stl'))
#sys.path.append(additional_stl_udp_pkts)
#from udp_1pkt_vxlan import VXLAN
#sys.path.remove(additional_stl_udp_pkts)
try:
from cStringIO import StringIO
except ImportError:
from io import StringIO
class Scapy_service_api():
def get_version_handler(self,client_v_major,client_v_minor):
""" get_version_handler(self,client_v_major,client_v_minor)
Gives a handler to client to connect and use server api
Parameters
----------
client_v_major - major number of api version on the client side
Returns
-------
Handler(string) to provide when using server api
"""
pass
def get_all(self,client_v_handler):
""" get_all(self,client_v_handler)
Sends all the protocols and fields that Scapy Service supports.
also sends the md5 of the Protocol DB and Fields DB used to check if the DB's are up to date
Parameters
----------
None
Returns
-------
Dictionary (of protocol DB and scapy fields DB)
Raises
------
Raises an exception when a DB error occurs (i.e a layer is not loaded properly and has missing components)
"""
pass
def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
""" check_update_of_dbs(self,client_v_handler,db_md5,field_md5)
Checks if the Scapy Service running on the server has a newer version of the databases that the client has
Parameters
----------
db_md5 - The md5 that was delivered with the protocol database that the client owns, when first received at the client
field_md5 - The md5 that was delivered with the fields database that the client owns, when first received at the client
Returns
-------
True/False according the Databases version(determined by their md5)
Raises
------
Raises an exception (ScapyException) when protocol DB/Fields DB is not up to date
"""
pass
def build_pkt(self,client_v_handler,pkt_model_descriptor):
""" build_pkt(self,client_v_handler,pkt_model_descriptor) -> Dictionary (of Offsets,Show2 and Buffer)
Performs calculations on the given packet and returns results for that packet.
Parameters
----------
pkt_descriptor - An array of dictionaries describing a network packet
Returns
-------
- The packets offsets: each field in every layer is mapped inside the Offsets Dictionary
- The Show2: A description of each field and its value in every layer of the packet
- The Buffer: The Hexdump of packet encoded in base64
Raises
------
will raise an exception when the Scapy string format is illegal, contains syntax error, contains non-supported
protocl, etc.
"""
pass
def get_tree(self,client_v_handler):
""" get_tree(self) -> Dictionary describing an example of hierarchy in layers
Scapy service holds a tree of layers that can be stacked to a recommended packet
according to the hierarchy
Parameters
----------
None
Returns
-------
Returns an example hierarchy tree of layers that can be stacked to a packet
Raises
------
None
"""
pass
def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
""" reconstruct_pkt(self,client_v_handler,binary_pkt)
Makes a Scapy valid packet by applying changes to binary packet and returns all information returned in build_pkt
Parameters
----------
Source packet in binary_pkt, formatted in "base64" encoding
List of changes in model_descriptor
Returns
-------
All data provided in build_pkt:
show2 - detailed description of the packet
buffer - the packet presented in binary
offsets - the offset[in bytes] of each field in the packet
"""
pass
def read_pcap(self,client_v_handler,pcap_base64):
""" read_pcap(self,client_v_handler,pcap_base64)
Parses pcap file contents and returns an array with build_pkt information for each packet
Parameters
----------
binary pcap file in base64 encoding
Returns
-------
Array of build_pkt(packet)
"""
pass
def write_pcap(self,client_v_handler,packets_base64):
""" write_pcap(self,client_v_handler,packets_base64)
Writes binary packets to pcap file
Parameters
----------
array of binary packets in base64 encoding
Returns
-------
binary pcap file in base64 encoding
"""
pass
def get_definitions(self,client_v_handler, def_filter):
""" get_definitions(self,client_v_handler, def_filter)
Returns protocols and fields metadata of scapy service
Parameters
----------
def_filter - array of protocol names
Returns
-------
definitions for protocols
"""
pass
def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
""" get_payload_classes(self,client_v_handler, pkt_model_descriptor)
Returns an array of protocol classes, which normally can be used as a payload
Parameters
----------
pkt_model_descriptor - see build_pkt
Returns
-------
array of supported protocol classes
"""
pass
def is_python(version):
return version == sys.version_info[0]
def is_number(obj):
return isinstance(obj, numbers.Number)
def is_string(obj):
return type(obj) == str or type(obj).__name__ == 'unicode' # python3 doesn't have unicode type
def is_ascii_str(strval):
return strval and all(ord(ch) < 128 for ch in strval)
def is_ascii_bytes(buf):
return buf and all(byte < 128 for byte in buf)
def is_ascii(obj):
if is_bytes3(obj):
return is_ascii_bytes(obj)
else:
return is_ascii_str(obj)
def is_bytes3(obj):
# checks if obj is exactly bytes(always false for python2)
return is_python(3) and type(obj) == bytes
def str_to_bytes(strval):
return strval.encode("utf8")
def bytes_to_str(buf):
return buf.decode("utf8")
def b64_to_bytes(payload_base64):
# get bytes from base64 string(unicode)
return base64.b64decode(payload_base64)
def bytes_to_b64(buf):
# bytes to base64 string(unicode)
return base64.b64encode(buf).decode('ascii')
def get_sample_field_val(scapy_layer, fieldId):
# get some sample value for the field, to determine the value type
# use random or serialized value if default value is None
field_desc, current_val = scapy_layer.getfield_and_val(fieldId)
if current_val is not None:
return current_val
try:
# try to get some random value to determine type
return field_desc.randval()._fix()
except:
pass
try:
# try to serialize/deserialize
ltype = type(scapy_layer)
pkt = ltype(bytes(ltype()))
return pkt.getfieldval(fieldId)
except:
pass
def generate_random_bytes(sz, seed, start, end):
# generate bytes of specified range with a fixed seed and size
rnd = random.Random()
n = end - start + 1
if is_python(2):
rnd = random.Random(seed)
res = [start + int(rnd.random()*n) for _i in range(sz)]
return ''.join(chr(x) for x in res)
else:
rnd = random.Random()
# to generate same random sequence as 2.x
rnd.seed(seed, version=1)
res = [start + int(rnd.random()*n) for _i in range(sz)]
return bytes(res)
def generate_bytes_from_template(sz, template):
# generate bytes by repeating a template
res = str_to_bytes('') # new bytes array
if len(template) == 0:
return res
while len(res) < sz:
res = res + template
return res[:sz]
def parse_template_code(template_code):
template_code = re.sub("0[xX]", '', template_code) # remove 0x
template_code = re.sub("[\s]", '', template_code) # remove spaces
return bytearray.fromhex(template_code)
def verify_payload_size(size):
assert(size != None)
if (size > (1<<20)): # 1Mb ought to be enough for anybody
raise ValueError('size is too large')
def generate_bytes(bytes_definition):
# accepts a bytes definition object
# {generate: random_bytes or random_ascii, seed: <seed_number>, size: <size_bytes>}
# {generate: template, template_base64: '<base64str>', size: <size_bytes>}
# {generate: template_code, template_text_code: '<template_code_str>', size: <size_bytes>}
gen_type = bytes_definition.get('generate')
if gen_type == None:
return b64_to_bytes(bytes_definition['base64'])
elif gen_type == 'template_code':
code = parse_template_code(bytes_definition["template_code"])
bytes_size = int(bytes_definition.get('size') or len(code))
verify_payload_size(bytes_size)
return generate_bytes_from_template(bytes_size, code)
else:
bytes_size = int(bytes_definition['size']) # required
seed = int(bytes_definition.get('seed') or 12345) # optional
verify_payload_size(bytes_size)
if gen_type == 'random_bytes':
return generate_random_bytes(bytes_size, seed, 0, 0xFF)
elif gen_type == 'random_ascii':
return generate_random_bytes(bytes_size, seed, 0x20, 0x7E)
elif gen_type == 'template':
return generate_bytes_from_template(bytes_size, b64_to_bytes(bytes_definition["template_base64"]))
class ScapyException(Exception): pass
class Scapy_service(Scapy_service_api):
#----------------------------------------------------------------------------------------------------
class ScapyFieldDesc:
def __init__(self,FieldName,regex='empty'):
self.FieldName = FieldName
self.regex = regex
#defualt values - should be changed when needed, or added to constructor
self.string_input =""
self.string_input_mex_len = 1
self.integer_input = 0
self.integer_input_min = 0
self.integer_input_max = 1
self.input_array = []
self.input_list_max_len = 1
def stringRegex(self):
return self.regex
#----------------------------------------------------------------------------------------------------
def __init__(self):
self.Raw = {'Raw':''}
self.high_level_protocols = ['Raw']
self.transport_protocols = {'TCP':self.Raw,'UDP':self.Raw}
self.network_protocols = {'IP':self.transport_protocols ,'ARP':''}
self.low_level_protocols = { 'Ether': self.network_protocols }
self.regexDB= {'MACField' : self.ScapyFieldDesc('MACField','^([0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])$'),
'IPField' : self.ScapyFieldDesc('IPField','^(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])$')}
self.all_protocols = self._build_lib()
self.protocol_tree = {'ALL':{'Ether':{'ARP':{},'IP':{'TCP':{'RAW':'payload'},'UDP':{'RAW':'payload'}}}}}
self.version_major = '1'
self.version_minor = '01'
self.server_v_hashed = self._generate_version_hash(self.version_major,self.version_minor)
self.protocol_definitions = {} # protocolId -> prococol definition overrides data
self._load_definitions_from_json()
def _load_definitions_from_json(self):
# load protocol definitions from a json file
self.protocol_definitions = {}
with open('protocols.json', 'r') as f:
protocols = json.load(f)
for protocol in protocols:
self.protocol_definitions[ protocol['id'] ] = protocol
def _all_protocol_structs(self):
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
ls()
sys.stdout = old_stdout
all_protocol_data= mystdout.getvalue()
return all_protocol_data
def _protocol_struct(self,protocol):
if '_' in protocol:
return []
if not protocol=='':
if protocol not in self.all_protocols:
return 'protocol not supported'
protocol = eval(protocol)
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
ls(protocol)
sys.stdout = old_stdout
protocol_data= mystdout.getvalue()
return protocol_data
def _build_lib(self):
lib = self._all_protocol_structs()
lib = lib.splitlines()
all_protocols=[]
for entry in lib:
entry = entry.split(':')
all_protocols.append(entry[0].strip())
del all_protocols[len(all_protocols)-1]
return all_protocols
def _parse_description_line(self,line):
line_arr = [x.strip() for x in re.split(': | = ',line)]
return tuple(line_arr)
def _parse_entire_description(self,description):
description = description.split('\n')
description_list = [self._parse_description_line(x) for x in description]
del description_list[len(description_list)-1]
return description_list
def _get_protocol_details(self,p_name):
protocol_str = self._protocol_struct(p_name)
if protocol_str=='protocol not supported':
return 'protocol not supported'
if len(protocol_str) is 0:
return []
tupled_protocol = self._parse_entire_description(protocol_str)
return tupled_protocol
def _value_from_dict(self, val):
# allows building python objects from json
if type(val) == type({}):
value_type = val['vtype']
if value_type == 'EXPRESSION':
return eval(val['expr'], scapy.all.__dict__)
elif value_type == 'BYTES': # bytes payload(ex Raw.load)
return generate_bytes(val)
elif value_type == 'OBJECT':
return val['value']
else:
return val # it's better to specify type explicitly
elif type(val) == type([]):
return [self._value_from_dict(v) for v in val]
else:
return val
def _field_value_from_def(self, scapy_pkt, layer, fieldId, val):
field_desc = layer.get_field(fieldId)
sample_val = get_sample_field_val(layer, fieldId)
# extensions for field values
if type(val) == type({}):
value_type = val['vtype']
if value_type == 'UNDEFINED': # clear field value
return None
elif value_type == 'RANDOM': # random field value
return field_desc.randval()
elif value_type == 'MACHINE': # internal machine field repr
return field_desc.m2i(layer, b64_to_bytes(val['base64']))
elif value_type == 'BYTES':
if 'total_size' in val: # custom case for total pkt size
gen = {}
gen.update(val)
total_sz = gen['total_size']
del gen['total_size']
gen['size'] = total_sz - len(scapy_pkt)
return generate_bytes(gen)
else:
return generate_bytes(val)
if is_number(sample_val) and is_string(val):
# human-value. guess the type and convert to internal value
# seems setfieldval already does this for some fields,
# but does not convert strings/hex(0x123) to integers and long
val = str(val) # unicode -> str(ascii)
# parse str to int/long as a decimal or hex
val_constructor = type(sample_val)
if len(val) == 0:
return None
elif re.match(r"^0x[\da-f]+$", val, flags=re.IGNORECASE): # hex
return val_constructor(val, 16)
elif re.match(r"^\d+L?$", val): # base10
return val_constructor(val)
# generate recursive field-independent values
return self._value_from_dict(val)
def _print_tree(self):
pprint(self.protocol_tree)
def _get_all_db(self):
db = {}
for pro in self.all_protocols:
details = self._get_protocol_details(pro)
db[pro] = details
return db
def _get_all_fields(self):
fields = []
for pro in self.all_protocols:
details = self._get_protocol_details(pro)
for i in range(0,len(details),1):
if len(details[i]) == 3:
fields.append(details[i][1])
uniqueFields = list(set(fields))
fieldDict = {}
for f in uniqueFields:
if f in self.regexDB:
fieldDict[f] = self.regexDB[f].stringRegex()
else:
fieldDict[f] = self.ScapyFieldDesc(f).stringRegex()
return fieldDict
def _fully_define(self,pkt):
# returns scapy object with all fields initialized
rootClass = type(pkt)
full_pkt = rootClass(bytes(pkt))
full_pkt.build() # this trick initializes offset
return full_pkt
def _bytes_to_value(self, payload_bytes):
# generates struct with a value
return { "vtype": "BYTES", "base64": bytes_to_b64(payload_bytes) }
def _pkt_to_field_tree(self,pkt):
pkt.build()
result = []
pcap_struct = self._fully_define(pkt) # structure, which will appear in pcap binary
while pkt:
layer_id = type(pkt).__name__ # Scapy classname
layer_full = self._fully_define(pkt) # current layer recreated from binary to get auto-calculated vals
real_layer_id = type(pcap_struct).__name__ if pcap_struct else None
valid_struct = True # shows if packet is mapped correctly to the binary representation
if not pcap_struct:
valid_struct = False
elif not issubclass(type(pkt), type(pcap_struct)) and not issubclass(type(pcap_struct), type(pkt)):
# structure mismatch. no need to go deeper in pcap_struct
valid_struct = False
pcap_struct = None
fields = []
for field_desc in pkt.fields_desc:
field_id = field_desc.name
ignored = field_id not in layer_full.fields
offset = field_desc.offset
protocol_offset = pkt.offset
field_sz = field_desc.get_size_bytes()
# some values are unavailable in pkt(original model)
# at the same time,
fieldval = pkt.getfieldval(field_id)
pkt_fieldval_defined = is_string(fieldval) or is_number(fieldval) or is_bytes3(fieldval)
if not pkt_fieldval_defined:
fieldval = layer_full.getfieldval(field_id)
value = None
hvalue = None
value_base64 = None
if is_python(3) and is_bytes3(fieldval):
value = self._bytes_to_value(fieldval)
if is_ascii_bytes(fieldval):
hvalue = bytes_to_str(fieldval)
else:
# can't be shown as ascii.
# also this buffer may not be unicode-compatible(still can try to convert)
value = self._bytes_to_value(fieldval)
hvalue = '<binary>'
elif not is_string(fieldval):
# value as is. this can be int,long, or custom object(list/dict)
# "nice" human value, i2repr(string) will have quotes, so we have special handling for them
hvalue = field_desc.i2repr(pkt, fieldval)
if is_number(fieldval):
value = fieldval
if is_string(hvalue) and re.match(r"^\d+L$", hvalue):
hvalue = hvalue[:-1] # chop trailing L for long decimal number(python2)
else:
# fieldval is an object( class / list / dict )
# generic serialization/deserialization needed for proper packet rebuilding from packet tree,
# some classes can not be mapped to json, but we can pass them serialize them
# as a python eval expr, value bytes base64, or field machine internal val(m2i)
value = {"vtype": "EXPRESSION", "expr": hvalue}
if is_python(3) and is_string(fieldval):
hvalue = value = fieldval
if is_python(2) and is_string(fieldval):
if is_ascii(fieldval):
hvalue = value = fieldval
else:
# python2 non-ascii byte buffers
# payload contains non-ascii chars, which
# sometimes can not be passed as unicode strings
value = self._bytes_to_value(fieldval)
hvalue = '<binary>'
if field_desc.name == 'load':
# show Padding(and possible similar classes) as Raw
layer_id = 'Raw'
field_sz = len(pkt)
value = self._bytes_to_value(fieldval)
field_data = {
"id": field_id,
"value": value,
"hvalue": hvalue,
"offset": offset,
"length": field_sz
}
if ignored:
field_data["ignored"] = ignored
fields.append(field_data)
layer_data = {
"id": layer_id,
"offset": pkt.offset,
"fields": fields,
"real_id": real_layer_id,
"valid_structure": valid_struct,
}
result.append(layer_data)
pkt = pkt.payload
if pcap_struct:
pcap_struct = pcap_struct.payload or None
return result
#input: container
#output: md5 encoded in base64
def _get_md5(self,container):
container = json.dumps(container)
m = hashlib.md5()
m.update(str_to_bytes(container))
res_md5 = bytes_to_b64(m.digest())
return res_md5
def get_version(self):
return {'built_by':'itraviv','version':self.version_major+'.'+self.version_minor}
def supported_methods(self,method_name='all'):
if method_name=='all':
methods = {}
for f in dir(Scapy_service):
if f[0]=='_':
continue
if inspect.ismethod(eval('Scapy_service.'+f)):
param_list = inspect.getargspec(eval('Scapy_service.'+f))[0]
del param_list[0] #deleting the parameter "self" that appears in every method
#because the server automatically operates on an instance,
#and this can cause confusion
methods[f] = (len(param_list), param_list)
return methods
if method_name in dir(Scapy_service):
return True
return False
def _generate_version_hash(self,v_major,v_minor):
v_for_hash = v_major+v_minor+v_major+v_minor
m = hashlib.md5()
m.update(str_to_bytes(v_for_hash))
return bytes_to_b64(m.digest())
def _generate_invalid_version_error(self):
error_desc1 = "Provided version handler does not correspond to the server's version.\nUpdate client to latest version.\nServer version:"+self.version_major+"."+self.version_minor
return error_desc1
def _verify_version_handler(self,client_v_handler):
return (self.server_v_hashed == client_v_handler)
def _parse_packet_dict(self, layer, layer_classes, base_layer):
class_p = layer_classes[layer['id']] # class id -> class dict
scapy_layer = class_p()
if isinstance(scapy_layer, Raw):
scapy_layer.load = str_to_bytes("dummy")
if base_layer == None:
base_layer = scapy_layer
if 'fields' in layer:
self._modify_layer(base_layer, scapy_layer, layer['fields'])
return scapy_layer
def _packet_model_to_scapy_packet(self,data):
layer_classes = {}
for layer_class in Packet.__subclasses__():
layer_classes[layer_class.__name__] = layer_class
base_layer = self._parse_packet_dict(data[0], layer_classes, None)
for i in range(1,len(data),1):
packet_layer = self._parse_packet_dict(data[i], layer_classes, base_layer)
base_layer = base_layer/packet_layer
return base_layer
def _pkt_data(self,pkt):
if pkt == None:
return {'data': [], 'binary': None}
data = self._pkt_to_field_tree(pkt)
binary = bytes_to_b64(bytes(pkt))
res = {'data': data, 'binary': binary}
return res
#--------------------------------------------API implementation-------------
def get_tree(self,client_v_handler):
if not (self._verify_version_handler(client_v_handler)):
raise ScapyException(self._generate_invalid_version_error())
return self.protocol_tree
def get_version_handler(self,client_v_major,client_v_minor):
v_handle = self._generate_version_hash(client_v_major,client_v_minor)
return v_handle
# pkt_descriptor in packet model format (dictionary)
def build_pkt(self,client_v_handler,pkt_model_descriptor):
if not (self._verify_version_handler(client_v_handler)):
raise ScapyException(self._generate_invalid_version_error())
pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
return self._pkt_data(pkt)
# @deprecated. to be removed
def get_all(self,client_v_handler):
if not (self._verify_version_handler(client_v_handler)):
raise ScapyException(self._generate_invalid_version_error())
fields=self._get_all_fields()
db=self._get_all_db()
fields_md5 = self._get_md5(fields)
db_md5 = self._get_md5(db)
res = {}
res['db'] = db
res['fields'] = fields
res['db_md5'] = db_md5
res['fields_md5'] = fields_md5
return res
def _is_packet_class(self, pkt_class):
# returns true for final Packet classes. skips aliases and metaclasses
return issubclass(pkt_class, Packet) and pkt_class.name and pkt_class.fields_desc
def _getDummyPacket(self, pkt_class):
if issubclass(pkt_class, Raw):
# need to have some payload. otherwise won't appear in the binary chunk
return pkt_class(load=str_to_bytes("dummy"))
else:
return pkt_class()
def _get_payload_classes(self, pkt_class):
# tries to find, which subclasses allowed.
# this can take long time, since it tries to build packets with all subclasses(O(N))
allowed_subclasses = []
for pkt_subclass in conf.layers:
if self._is_packet_class(pkt_subclass):
try:
pkt_w_payload = pkt_class() / self._getDummyPacket(pkt_subclass)
recreated_pkt = pkt_class(bytes(pkt_w_payload))
if type(recreated_pkt.lastlayer()) is pkt_subclass:
allowed_subclasses.append(pkt_subclass)
except Exception as e:
# no actions needed on fail, just sliently skip
pass
return allowed_subclasses
def _get_fields_definition(self, pkt_class, fieldsDef):
# fieldsDef - array of field definitions(or empty array)
fields = []
for field_desc in pkt_class.fields_desc:
fieldId = field_desc.name
field_data = {
"id": fieldId,
"name": field_desc.name
}
for fieldDef in fieldsDef:
if fieldDef['id'] == fieldId:
field_data.update(fieldDef)
if isinstance(field_desc, EnumField):
try:
field_data["values_dict"] = field_desc.s2i
if field_data.get("type") == None:
if len(field_data["values_dict"] > 0):
field_data["type"] = "ENUM"
elif fieldId == 'load':
field_data["type"] = "BYTES"
else:
field_data["type"] = "STRING"
field_data["values_dict"] = field_desc.s2i
except:
# MultiEnumField doesn't have s2i. need better handling
pass
fields.append(field_data)
return fields
def get_definitions(self,client_v_handler, def_filter):
# def_filter is an array of classnames or None
all_classes = Packet.__subclasses__() # as an alternative to conf.layers
if def_filter:
all_classes = [c for c in all_classes if c.__name__ in def_filter]
protocols = []
for pkt_class in all_classes:
if self._is_packet_class(pkt_class):
# enumerate all non-abstract Packet classes
protocolId = pkt_class.__name__
protoDef = self.protocol_definitions.get(protocolId) or {}
protocols.append({
"id": protocolId,
"name": protoDef.get('name') or pkt_class.name,
"fields": self._get_fields_definition(pkt_class, protoDef.get('fields') or [])
})
res = {"protocols": protocols}
return res
def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
pkt_class = type(pkt.lastlayer())
protocolDef = self.protocol_definitions.get(pkt_class.__name__)
if protocolDef and protocolDef.get('payload'):
return protocolDef['payload']
return [c.__name__ for c in self._get_payload_classes(pkt_class)]
#input in string encoded base64
def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
if not (self._verify_version_handler(client_v_handler)):
raise ScapyException(self._generate_invalid_version_error())
fields=self._get_all_fields()
db=self._get_all_db()
current_db_md5 = self._get_md5(db)
current_field_md5 = self._get_md5(fields)
res = []
if (field_md5 == current_field_md5):
if (db_md5 == current_db_md5):
return True
else:
raise ScapyException("Protocol DB is not up to date")
else:
raise ScapyException("Fields DB is not up to date")
def _modify_layer(self, scapy_pkt, scapy_layer, fields):
for field in fields:
fieldId = str(field['id'])
fieldval = self._field_value_from_def(scapy_pkt, scapy_layer, fieldId, field['value'])
if fieldval is not None:
scapy_layer.setfieldval(fieldId, fieldval)
else:
scapy_layer.delfieldval(fieldId)
def _is_last_layer(self, layer):
# can be used, that layer has no payload
# if true, the layer.payload is likely NoPayload()
return layer is layer.lastlayer()
#input of binary_pkt must be encoded in base64
def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
pkt_bin = b64_to_bytes(binary_pkt)
scapy_pkt = Ether(pkt_bin)
if not model_descriptor:
model_descriptor = []
for depth in range(len(model_descriptor)):
model_layer = model_descriptor[depth]
if model_layer.get('delete') is True:
# slice packet from the current item
if depth == 0:
scapy_pkt = None
break
else:
scapy_pkt[depth-1].payload = None
break
if depth > 0 and self._is_last_layer(scapy_pkt[depth-1]):
# insert new layer(s) from json definition
remaining_definitions = model_descriptor[depth:]
pkt_to_append = self._packet_model_to_scapy_packet(remaining_definitions)
scapy_pkt = scapy_pkt / pkt_to_append
break
# modify fields of existing stack items
scapy_layer = scapy_pkt[depth]
if model_layer['id'] != type(scapy_layer).__name__:
# TODO: support replacing payload, instead of breaking
raise ScapyException("Protocol id inconsistent")
if 'fields' in model_layer:
self._modify_layer(scapy_pkt, scapy_layer, model_layer['fields'])
return self._pkt_data(scapy_pkt)
def read_pcap(self,client_v_handler,pcap_base64):
pcap_bin = b64_to_bytes(pcap_base64)
pcap = []
res_packets = []
with tempfile.NamedTemporaryFile(mode='w+b') as tmpPcap:
tmpPcap.write(pcap_bin)
tmpPcap.flush()
pcap = rdpcap(tmpPcap.name)
for scapy_packet in pcap:
res_packets.append(self._pkt_data(scapy_packet))
return res_packets
def write_pcap(self,client_v_handler,packets_base64):
packets = [Ether(b64_to_bytes(pkt_b64)) for pkt_b64 in packets_base64]
pcap_bin = None
with tempfile.NamedTemporaryFile(mode='r+b') as tmpPcap:
wrpcap(tmpPcap.name, packets)
pcap_bin = tmpPcap.read()
return bytes_to_b64(pcap_bin)
#---------------------------------------------------------------------------
|