path: root/scripts/automation/trex_control_plane/stl/services/scapy_server/scapy_service.py

import os
import sys

stl_pathname = os.path.abspath(os.path.join(os.pardir, os.pardir))
sys.path.append(stl_pathname)

from trex_stl_lib.api import *
import trex_stl_lib.trex_stl_packet_builder_scapy
import tempfile
import hashlib
import base64
import numbers
import random
from inspect import getdoc
import json
import re
from pprint import pprint

# add some layers as an example
# need to test more 
from scapy.layers.dns import *
from scapy.layers.dhcp import *
from scapy.layers.ipsec import *
from scapy.layers.netflow import *
from scapy.layers.sctp import *
from scapy.layers.tftp import *

from scapy.contrib.mpls import *
from scapy.contrib.igmp import *
from scapy.contrib.igmpv3 import *




#additional_stl_udp_pkts = os.path.abspath(os.path.join(os.pardir,os.pardir,os.pardir,os.pardir, os.pardir,'stl'))
#sys.path.append(additional_stl_udp_pkts)
#from udp_1pkt_vxlan import VXLAN
#sys.path.remove(additional_stl_udp_pkts)

try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO




class Scapy_service_api():

    def get_version_handler(self,client_v_major,client_v_minor):
        """ get_version_handler(self,client_v_major,client_v_minor)
            
            Gives a handler to client to connect and use server api

            Parameters
            ----------
            client_v_major - major number of api version on the client side

            Returns
            -------
            Handler(string) to provide when using server api
        """
        pass
    def get_all(self,client_v_handler):
        """ get_all(self,client_v_handler) 

        Sends all the protocols and fields that Scapy Service supports.
        also sends the md5 of the Protocol DB and Fields DB used to check if the DB's are up to date

        Parameters
        ----------
        None

        Returns
        -------
        Dictionary (of protocol DB and scapy fields DB)

        Raises
        ------
        Raises an exception when a DB error occurs (i.e a layer is not loaded properly and has missing components)
        """
        pass

    def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):        
        """ check_update_of_dbs(self,client_v_handler,db_md5,field_md5) 
        Checks if the Scapy Service running on the server has a newer version of the databases that the client has

        Parameters
        ----------
        db_md5 - The md5 that was delivered with the protocol database that the client owns, when first received at the client
        field_md5 - The md5 that was delivered with the fields database that the client owns, when first received at the client

        Returns
        -------
        True/False according the Databases version(determined by their md5)

        Raises
        ------
        Raises an exception (ScapyException) when protocol DB/Fields DB is not up to date
        """
        pass


    def build_pkt(self,client_v_handler,pkt_model_descriptor):
        """ build_pkt(self,client_v_handler,pkt_model_descriptor) -> Dictionary (of Offsets,Show2 and Buffer)
        
        Performs calculations on the given packet and returns results for that packet.
    
        Parameters
        ----------
        pkt_descriptor - An array of dictionaries describing a network packet

        Returns
        -------
        - The packets offsets: each field in every layer is mapped inside the Offsets Dictionary
        - The Show2: A description of each field and its value in every layer of the packet
        - The Buffer: The Hexdump of packet encoded in base64

        Raises
        ------
        will raise an exception when the Scapy string format is illegal, contains syntax error, contains non-supported
        protocl, etc.
        """
        pass

    def build_pkt_ex(self, client_v_handler, pkt_model_descriptor, extra_options):
        """ build_pkt_ex(self,client_v_handler,pkt_model_descriptor, extra_options) -> Dictionary (of Offsets,Show2 and Buffer)
        Performs calculations on the given packet and returns results for that packet.

        Parameters
        ----------
        pkt_descriptor - An array of dictionaries describing a network packet
        extra_options - A dictionary of extra options required for building packet

        Returns
        -------
        - The packets offsets: each field in every layer is mapped inside the Offsets Dictionary
        - The Show2: A description of each field and its value in every layer of the packet
        - The Buffer: The Hexdump of packet encoded in base64

        Raises
        ------
        will raise an exception when the Scapy string format is illegal, contains syntax error, contains non-supported
        protocl, etc.
        """
        pass

    def load_instruction_parameter_values(self, client_v_handler, pkt_model_descriptor, vm_instructions_model, parameter_id):
        """ load_instruction_parameter_values(self,client_v_handler,pkt_model_descriptor, vm_instructions_model, parameter_id) -> Dictionary (of possible parameter values)
        Returns possible valies for given pararameter id depends on current pkt structure and vm_instructions
        model.

        Parameters
        ----------
        pkt_descriptor - An array of dictionaries describing a network packet
        vm_instructions_model - A dictionary of extra options required for building packet
        parameter_id - A string of parameter id

        Returns
        -------
        Possible parameter values map.

        """
        pass

    def get_tree(self,client_v_handler):
        """ get_tree(self) -> Dictionary describing an example of hierarchy in layers

        Scapy service holds a tree of layers that can be stacked to a recommended packet
        according to the hierarchy

        Parameters
        ----------
        None

        Returns
        -------
        Returns an example hierarchy tree of layers that can be stacked to a packet

        Raises
        ------
        None
        """
        pass
    
    def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
        """ reconstruct_pkt(self,client_v_handler,binary_pkt)

        Makes a Scapy valid packet by applying changes to binary packet and returns all information returned in build_pkt

        Parameters
        ----------
        Source packet in binary_pkt, formatted in "base64" encoding
        List of changes in model_descriptor

        Returns
        -------
        All data provided in build_pkt:
        show2 - detailed description of the packet
        buffer - the packet presented in binary
        offsets - the offset[in bytes] of each field in the packet

        """
        pass

    def read_pcap(self,client_v_handler,pcap_base64):
        """ read_pcap(self,client_v_handler,pcap_base64)

        Parses pcap file contents and returns an array with build_pkt information for each packet

        Parameters
        ----------
        binary pcap file in base64 encoding

        Returns
        -------
        Array of build_pkt(packet)
        """
        pass

    def write_pcap(self,client_v_handler,packets_base64):
        """ write_pcap(self,client_v_handler,packets_base64)

        Writes binary packets to pcap file

        Parameters
        ----------
        array of binary packets in base64 encoding

        Returns
        -------
        binary pcap file in base64 encoding
        """
        pass

    def get_definitions(self,client_v_handler, def_filter):
        """ get_definitions(self,client_v_handler, def_filter)

        Returns protocols and fields metadata of scapy service

        Parameters
        ----------
        def_filter - array of protocol names

        Returns
        -------
        definitions for protocols
        """
        pass

    def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
        """ get_payload_classes(self,client_v_handler, pkt_model_descriptor)

        Returns an array of protocol classes, which normally can be used as a payload

        Parameters
        ----------
        pkt_model_descriptor - see build_pkt

        Returns
        -------
        array of supported protocol classes
        """
        pass

    def get_templates(self,client_v_handler):
        """ get_templates(self,client_v_handler)

        Returns an array of templates, which normally can be used for creating packet

        Parameters
        ----------

        Returns
        -------
        array of templates
        """
        pass

    def get_template(self,client_v_handler,template):
        """ get_template(self,client_v_handler,template)

        Returns a template, which normally can be used for creating packet

        Parameters
        ----------

        Returns
        -------
        base64 of template content
        """
        pass

def is_python(version):
    return version == sys.version_info[0]

def is_number(obj):
    return isinstance(obj, numbers.Number)

def is_string(obj):
    return type(obj) == str or type(obj).__name__ == 'unicode' # python3 doesn't have unicode type

def is_ascii_str(strval):
    return strval and all(ord(ch) < 128 for ch in strval)

def is_ascii_bytes(buf):
    return buf and all(byte < 128 for byte in buf)

def is_ascii(obj):
    if is_bytes3(obj):
        return is_ascii_bytes(obj)
    else:
        return is_ascii_str(obj)

def is_bytes3(obj):
    # checks if obj is exactly bytes(always false for python2)
    return is_python(3) and type(obj) == bytes

def str_to_bytes(strval):
    return strval.encode("utf8")

def bytes_to_str(buf):
    return buf.decode("utf8")

def b64_to_bytes(payload_base64):
    # get bytes from base64 string(unicode)
    return base64.b64decode(payload_base64)

def bytes_to_b64(buf):
    # bytes to base64 string(unicode)
    return base64.b64encode(buf).decode('ascii')

def get_sample_field_val(scapy_layer, fieldId):
    # get some sample value for the field, to determine the value type
    # use random or serialized value if default value is None
    field_desc, current_val = scapy_layer.getfield_and_val(fieldId)
    if current_val is not None:
        return current_val
    try:
        # try to get some random value to determine type
        return field_desc.randval()._fix()
    except:
        pass
    try:
        # try to serialize/deserialize
        ltype = type(scapy_layer)
        pkt = ltype(bytes(ltype()))
        return pkt.getfieldval(fieldId)
    except:
        pass

def generate_random_bytes(sz, seed, start, end):
    # generate bytes of specified range with a fixed seed and size
    rnd = random.Random()
    n = end - start + 1
    if is_python(2):
        rnd = random.Random(seed)
        res = [start + int(rnd.random()*n) for _i in range(sz)]
        return ''.join(chr(x) for x in res)
    else:
        rnd = random.Random()
        # to generate same random sequence as 2.x
        rnd.seed(seed, version=1)
        res = [start + int(rnd.random()*n) for _i in range(sz)]
        return bytes(res)

def generate_bytes_from_template(sz, template):
    # generate bytes by repeating a template
    res = str_to_bytes('') # new bytes array
    if len(template) == 0:
        return res
    while len(res) < sz:
        res = res + template
    return res[:sz]

def parse_template_code(template_code):
    template_code = re.sub("0[xX]", '', template_code) # remove 0x
    template_code = re.sub("[\s]", '', template_code) # remove spaces
    return bytearray.fromhex(template_code)

def verify_payload_size(size):
    assert(size != None)
    if (size > (1<<20)): # 1Mb ought to be enough for anybody
        raise ValueError('size is too large')

def generate_bytes(bytes_definition):
    # accepts a bytes definition object
    # {generate: random_bytes or random_ascii, seed: <seed_number>, size: <size_bytes>}
    # {generate: template, template_base64: '<base64str>',  size: <size_bytes>}
    # {generate: template_code, template_text_code: '<template_code_str>',  size: <size_bytes>}
    gen_type = bytes_definition.get('generate')
    if gen_type == None:
        return b64_to_bytes(bytes_definition['base64'])
    elif gen_type == 'template_code':
        code = parse_template_code(bytes_definition["template_code"])
        bytes_size = int(bytes_definition.get('size') or len(code))
        verify_payload_size(bytes_size)
        return generate_bytes_from_template(bytes_size, code)
    else:
        bytes_size = int(bytes_definition['size']) # required
        seed = int(bytes_definition.get('seed') or 12345) # optional
        verify_payload_size(bytes_size)
        if gen_type == 'random_bytes':
            return generate_random_bytes(bytes_size, seed, 0, 0xFF)
        elif gen_type == 'random_ascii':
            return generate_random_bytes(bytes_size, seed, 0x20, 0x7E)
        elif gen_type == 'template':
            return generate_bytes_from_template(bytes_size, b64_to_bytes(bytes_definition["template_base64"]))

class ScapyException(Exception): pass
class Scapy_service(Scapy_service_api):

#----------------------------------------------------------------------------------------------------
    class ScapyFieldDesc:
        def __init__(self,FieldName,regex='empty'):
            self.FieldName = FieldName
            self.regex = regex
            #defualt values - should be changed when needed, or added to constructor
            self.string_input =""
            self.string_input_mex_len = 1
            self.integer_input = 0
            self.integer_input_min = 0
            self.integer_input_max = 1
            self.input_array = []
            self.input_list_max_len = 1

        def stringRegex(self):
            return self.regex        
#----------------------------------------------------------------------------------------------------
    def __init__(self):
        self.Raw = {'Raw':''}
        self.high_level_protocols = ['Raw']
        self.transport_protocols = {'TCP':self.Raw,'UDP':self.Raw}
        self.network_protocols = {'IP':self.transport_protocols ,'ARP':''}
        self.low_level_protocols = { 'Ether': self.network_protocols }
        self.regexDB= {'MACField' : self.ScapyFieldDesc('MACField','^([0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])$'),
              'IPField' : self.ScapyFieldDesc('IPField','^(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])$')}
        self.all_protocols = self._build_lib()
        self.protocol_tree = {'ALL':{'Ether':{'ARP':{},'IP':{'TCP':{'RAW':'payload'},'UDP':{'RAW':'payload'}}}}}
        self.version_major = '1'
        self.version_minor = '01'
        self.server_v_hashed = self._generate_version_hash(self.version_major,self.version_minor)
        self.protocol_definitions = {} # protocolId -> prococol definition overrides data
        self.field_engine_supported_protocols = {}
        self.instruction_parameter_meta_definitions = []
        self.field_engine_parameter_meta_definitions = []
        self.field_engine_templates_definitions = []
        self.field_engine_instructions_meta = []
        self.field_engine_instruction_expressions = []
        self._load_definitions_from_json()
        self._load_field_engine_meta_from_json()
        self._vm_instructions = dict([m for m in inspect.getmembers(trex_stl_lib.trex_stl_packet_builder_scapy, inspect.isclass) if m[1].__module__ == 'trex_stl_lib.trex_stl_packet_builder_scapy'])

    def _load_definitions_from_json(self):
        # load protocol definitions from a json file
        self.protocol_definitions = {}
        with open('protocols.json', 'r') as f:
            protocols = json.load(f)
            for protocol in protocols:
                self.protocol_definitions[ protocol['id'] ] = protocol

    def _load_field_engine_meta_from_json(self):
        # load protocol definitions from a json file
        self.instruction_parameter_meta_definitions = []
        self.field_engine_supported_protocols = {}
        self.field_engine_parameter_meta_definitions = []
        self.field_engine_templates_definitions = []
        with open('field_engine.json', 'r') as f:
            metas = json.load(f)
            self.instruction_parameter_meta_definitions = metas["instruction_params_meta"]
            self.field_engine_instructions_meta = metas["instructions"]
            self._append_intructions_help()
            self.field_engine_supported_protocols = metas["supported_protocols"]
            self.field_engine_parameter_meta_definitions = metas["global_params_meta"]
            self.field_engine_templates_definitions = metas["templates"]


    def _append_intructions_help(self):
        for instruction_meta in self.field_engine_instructions_meta:
            clazz = eval(instruction_meta['id'])
            instruction_meta['help'] = base64.b64encode(getdoc(clazz.__init__).encode()).decode('ascii')

    def _all_protocol_structs(self):
        old_stdout = sys.stdout
        sys.stdout = mystdout = StringIO()
        ls()
        sys.stdout = old_stdout
        all_protocol_data= mystdout.getvalue()
        return all_protocol_data

    def _protocol_struct(self,protocol):
        if '_' in protocol:
            return []
        if not protocol=='':
            if protocol not in self.all_protocols:
                return 'protocol not supported'
        protocol = eval(protocol)
        old_stdout = sys.stdout
        sys.stdout = mystdout = StringIO()
        ls(protocol)
        sys.stdout = old_stdout
        protocol_data= mystdout.getvalue()
        return protocol_data

    def _build_lib(self):
        lib = self._all_protocol_structs()
        lib = lib.splitlines()
        all_protocols=[]
        for entry in lib:
            entry = entry.split(':')
            all_protocols.append(entry[0].strip())
        del all_protocols[len(all_protocols)-1]
        return all_protocols

    def _parse_description_line(self,line):
        line_arr = [x.strip() for x in re.split(': | = ',line)]
        return tuple(line_arr)

    def _parse_entire_description(self,description):
        description = description.split('\n')
        description_list = [self._parse_description_line(x) for x in description]
        del description_list[len(description_list)-1]
        return description_list

    def _get_protocol_details(self,p_name):
        protocol_str = self._protocol_struct(p_name)
        if protocol_str=='protocol not supported':
            return 'protocol not supported'
        if len(protocol_str) is 0:
            return []
        tupled_protocol = self._parse_entire_description(protocol_str)
        return tupled_protocol

    def _value_from_dict(self, val):
        # allows building python objects from json
        if type(val) == type({}):
            value_type = val['vtype']
            if value_type == 'EXPRESSION':
                return eval(val['expr'], scapy.all.__dict__)
            elif value_type == 'BYTES':   # bytes payload(ex Raw.load)
                return generate_bytes(val)
            elif value_type == 'OBJECT':
                return val['value']
            else:
                return val # it's better to specify type explicitly
        elif type(val) == type([]):
            return [self._value_from_dict(v) for v in val]
        else:
            return val

    def _field_value_from_def(self, scapy_pkt, layer, fieldId, val):
        field_desc = layer.get_field(fieldId)
        sample_val = get_sample_field_val(layer, fieldId)
        # extensions for field values
        if type(val) == type({}):
            value_type = val['vtype']
            if value_type == 'UNDEFINED': # clear field value
                return None
            elif value_type == 'RANDOM': # random field value
                return field_desc.randval()
            elif value_type == 'MACHINE': # internal machine field repr
                return field_desc.m2i(layer, b64_to_bytes(val['base64']))
            elif value_type == 'BYTES':
                if 'total_size' in val: # custom case for total pkt size
                    gen = {}
                    gen.update(val)
                    total_sz = gen['total_size']
                    del gen['total_size']
                    ether_chksum_size_bytes = 4 # will be added outside of Scapy. needs to be excluded here
                    gen['size'] = total_sz - len(scapy_pkt) - ether_chksum_size_bytes
                    return generate_bytes(gen)
                else:
                    return generate_bytes(val)
        if is_number(sample_val) and is_string(val):
            # human-value. guess the type and convert to internal value
            # seems setfieldval already does this for some fields,
            # but does not convert strings/hex(0x123) to integers and long
            val = str(val) # unicode -> str(ascii)
            # parse str to int/long as a decimal or hex
            val_constructor = type(sample_val)
            if len(val) == 0:
                return None
            elif re.match(r"^0x[\da-f]+$", val, flags=re.IGNORECASE): # hex
                return val_constructor(val, 16)
            elif re.match(r"^\d+L?$", val): # base10
                return val_constructor(val)
        # generate recursive field-independent values
        return self._value_from_dict(val)

    def _print_tree(self):
        pprint(self.protocol_tree)

    def _get_all_db(self):
        db = {}
        for pro in self.all_protocols:
            details = self._get_protocol_details(pro)
            db[pro] = details
        return db

    def _get_all_fields(self):
        fields = []
        for pro in self.all_protocols:
            details = self._get_protocol_details(pro)
            for i in range(0,len(details),1):
                if len(details[i]) == 3:
                    fields.append(details[i][1])
        uniqueFields = list(set(fields))
        fieldDict = {}
        for f in uniqueFields:
            if f in self.regexDB:
                fieldDict[f] = self.regexDB[f].stringRegex()
            else:
                fieldDict[f] = self.ScapyFieldDesc(f).stringRegex()
        return fieldDict

    def _fully_define(self,pkt):
        # returns scapy object with all fields initialized
        rootClass = type(pkt)
        full_pkt = rootClass(bytes(pkt))
        full_pkt.build() # this trick initializes offset
        return full_pkt

    def _bytes_to_value(self, payload_bytes):
        # generates struct with a value
        return { "vtype": "BYTES", "base64": bytes_to_b64(payload_bytes) }

    def _pkt_to_field_tree(self,pkt):
        pkt.build()
        result = []
        pcap_struct = self._fully_define(pkt) # structure, which will appear in pcap binary
        while pkt:
            layer_id = type(pkt).__name__ # Scapy classname
            layer_full = self._fully_define(pkt) # current layer recreated from binary to get auto-calculated vals
            real_layer_id = type(pcap_struct).__name__ if pcap_struct else None
            valid_struct = True # shows if packet is mapped correctly to the binary representation
            if not pcap_struct:
                valid_struct = False
            elif not issubclass(type(pkt), type(pcap_struct)) and not issubclass(type(pcap_struct), type(pkt)):
                # structure mismatch. no need to go deeper in pcap_struct
                valid_struct = False
                pcap_struct = None
            fields = []
            for field_desc in pkt.fields_desc:
                field_id = field_desc.name
                ignored = field_id not in layer_full.fields
                offset = field_desc._offset
                protocol_offset = pkt._offset
                field_sz = field_desc.get_size_bytes()
                # some values are unavailable in pkt(original model)
                # at the same time,
                fieldval = pkt.getfieldval(field_id)
                pkt_fieldval_defined = is_string(fieldval) or is_number(fieldval) or is_bytes3(fieldval)
                if not pkt_fieldval_defined:
                    fieldval = layer_full.getfieldval(field_id)
                value = None
                hvalue = None
                value_base64 = None
                if is_python(3) and is_bytes3(fieldval):
                    value = self._bytes_to_value(fieldval)
                    if is_ascii_bytes(fieldval):
                        hvalue = bytes_to_str(fieldval)
                    else:
                        # can't be shown as ascii.
                        # also this buffer may not be unicode-compatible(still can try to convert)
                        value = self._bytes_to_value(fieldval)
                        hvalue = '<binary>'
                elif not is_string(fieldval):
                    # value as is. this can be int,long, or custom object(list/dict)
                    # "nice" human value, i2repr(string) will have quotes, so we have special handling for them
                    hvalue = field_desc.i2repr(pkt, fieldval)

                    if is_number(fieldval):
                        value = fieldval
                        if is_string(hvalue) and re.match(r"^\d+L$", hvalue):
                            hvalue =  hvalue[:-1] # chop trailing L for long decimal number(python2)
                    else:
                        # fieldval is an object( class / list / dict )
                        # generic serialization/deserialization needed for proper packet rebuilding from packet tree,
                        # some classes can not be mapped to json, but we can pass them serialize them
                        # as a python eval expr, value bytes base64, or field machine internal val(m2i)
                        value = {"vtype": "EXPRESSION", "expr": hvalue}
                if is_python(3) and is_string(fieldval):
                    hvalue = value = fieldval
                if is_python(2) and is_string(fieldval):
                    if is_ascii(fieldval):
                        hvalue = value = fieldval
                    else:
                        # python2 non-ascii byte buffers
                        # payload contains non-ascii chars, which
                        # sometimes can not be passed as unicode strings
                        value = self._bytes_to_value(fieldval)
                        hvalue = '<binary>'
                if field_desc.name == 'load':
                    # show Padding(and possible similar classes) as Raw
                    layer_id = 'Raw'
                    field_sz = len(pkt)
                    value = self._bytes_to_value(fieldval)
                field_data = {
                        "id": field_id,
                        "value": value,
                        "hvalue": hvalue,
                        "offset": offset,
                        "length": field_sz
                        }
                if ignored:
                    field_data["ignored"] = ignored
                fields.append(field_data)
            layer_data = {
                    "id": layer_id,
                    "offset": pkt._offset,
                    "fields": fields,
                    "real_id": real_layer_id,
                    "valid_structure": valid_struct,
                    }
            result.append(layer_data)
            pkt = pkt.payload
            if pcap_struct:
                pcap_struct = pcap_struct.payload or None
        return result

#input: container
#output: md5 encoded in base64
    def _get_md5(self,container):
        container = json.dumps(container)
        m = hashlib.md5()
        m.update(str_to_bytes(container))
        res_md5 = bytes_to_b64(m.digest())
        return res_md5

    def get_version(self):
        return {'built_by':'itraviv','version':self.version_major+'.'+self.version_minor}

    def supported_methods(self,method_name='all'):
        if method_name=='all':
            methods = {}
            for f in dir(Scapy_service):
                if f[0]=='_':
                    continue
                if inspect.ismethod(eval('Scapy_service.'+f)):
                    param_list = inspect.getargspec(eval('Scapy_service.'+f))[0]
                    del param_list[0] #deleting the parameter "self" that appears in every method 
                                      #because the server automatically operates on an instance,
                                      #and this can cause confusion
                    methods[f] = (len(param_list), param_list)
            return methods
        if method_name in dir(Scapy_service):
            return True
        return False

    def _generate_version_hash(self,v_major,v_minor):
        v_for_hash = v_major+v_minor+v_major+v_minor
        m = hashlib.md5()
        m.update(str_to_bytes(v_for_hash))
        return bytes_to_b64(m.digest())

    def _generate_invalid_version_error(self):
        error_desc1 = "Provided version handler does not correspond to the server's version.\nUpdate client to latest version.\nServer version:"+self.version_major+"."+self.version_minor
        return error_desc1

    def _verify_version_handler(self,client_v_handler):
        return (self.server_v_hashed == client_v_handler)

    def _parse_packet_dict(self, layer, layer_classes, base_layer):
        class_p = layer_classes[layer['id']] # class id -> class dict
        scapy_layer = class_p()
        if isinstance(scapy_layer, Raw):
            scapy_layer.load = str_to_bytes("dummy")
        if base_layer == None:
            base_layer = scapy_layer
        if 'fields' in layer:
            self._modify_layer(base_layer, scapy_layer, layer['fields'])
        return scapy_layer

    def _packet_model_to_scapy_packet(self,data):
        layer_classes = {}
        for layer_class in Packet.__subclasses__():
            layer_classes[layer_class.__name__] = layer_class
        base_layer = self._parse_packet_dict(data[0], layer_classes, None)
        for i in range(1,len(data),1):
            packet_layer = self._parse_packet_dict(data[i], layer_classes, base_layer)
            base_layer = base_layer/packet_layer
        return base_layer

    def _pkt_data(self,pkt):
        if pkt == None:
            return {'data': [], 'binary': None}
        data = self._pkt_to_field_tree(pkt)
        binary = bytes_to_b64(bytes(pkt))
        res = {'data': data, 'binary': binary}
        return res

#--------------------------------------------API implementation-------------
    def get_tree(self,client_v_handler):
        if not (self._verify_version_handler(client_v_handler)):
            raise ScapyException(self._generate_invalid_version_error())
        return self.protocol_tree

    def get_version_handler(self,client_v_major,client_v_minor):
        v_handle = self._generate_version_hash(client_v_major,client_v_minor)
        return v_handle

# pkt_descriptor in packet model format (dictionary)
    def build_pkt(self,client_v_handler,pkt_model_descriptor):
        if not (self._verify_version_handler(client_v_handler)):
            raise ScapyException(self._generate_invalid_version_error())
        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
        return self._pkt_data(pkt)


    def build_pkt_ex(self, client_v_handler, pkt_model_descriptor, extra_options):
        res = self.build_pkt(client_v_handler, pkt_model_descriptor)
        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)

        field_engine = {}
        field_engine['instructions'] = []
        field_engine['error'] = None
        try:
            field_engine['instructions'] = self._generate_vm_instructions(pkt, extra_options['field_engine'])
        except AssertionError as e:
            field_engine['error'] = e.message
        except CTRexPacketBuildException as e:
            field_engine['error'] = e.message

        field_engine['vm_instructions_expressions'] = self.field_engine_instruction_expressions
        res['field_engine'] = field_engine
        return res

    def load_instruction_parameter_values(self, client_v_handler, pkt_model_descriptor, vm_instructions_model, parameter_id):

        given_protocol_ids = [str(proto['id']) for proto in pkt_model_descriptor]

        values = {}
        if parameter_id == "name":
            values = self._curent_pkt_protocol_fields(given_protocol_ids, "_")

        if parameter_id == "fv_name":
            values = self._existed_flow_var_names(vm_instructions_model['field_engine']['instructions'])

        if parameter_id == "pkt_offset":
            values = self._curent_pkt_protocol_fields(given_protocol_ids, ".")

        if parameter_id == "offset":
            for ip_idx in range(given_protocol_ids.count("IP")):
                value = "IP:{0}".format(ip_idx)
                values[value] = value

        return {"map": values}

    def _existed_flow_var_names(self, instructions):
        return dict((instruction['parameters']['name'], instruction['parameters']['name']) for instruction in instructions if self._nameParamterExist(instruction))

    def _nameParamterExist(self, instruction):
        try:
            instruction['parameters']['name']
            return True
        except KeyError:
            return False

    def _curent_pkt_protocol_fields(self, given_protocol_ids, delimiter):
        given_protocol_classes = [c for c in Packet.__subclasses__() if c.__name__ in given_protocol_ids]
        protocol_fields = {}
        for protocol_class in given_protocol_classes:
            protocol_name = protocol_class.__name__
            protocol_count = given_protocol_ids.count(protocol_name)
            for field_desc in protocol_class.fields_desc:
                if delimiter == '.' and protocol_count > 1:
                    for idx in range(protocol_count):
                        formatted_name = "{0}:{1}{2}{3}".format(protocol_name, idx, delimiter, field_desc.name)
                        protocol_fields[formatted_name] = formatted_name
                else:
                    formatted_name = "{0}{1}{2}".format(protocol_name, delimiter, field_desc.name)
                protocol_fields[formatted_name] = formatted_name

        return protocol_fields

    def _generate_vm_instructions(self, pkt, field_engine_model_descriptor):
        self.field_engine_instruction_expressions = []
        instructions = []
        instructions_def = field_engine_model_descriptor['instructions']
        for instruction_def in instructions_def:
            instruction_id = instruction_def['id']
            instruction_class = self._vm_instructions[instruction_id]
            parameters = {k: self._sanitize_value(k, v) for (k, v) in instruction_def['parameters'].items()}
            instructions.append(instruction_class(**parameters))

        fe_parameters = field_engine_model_descriptor['global_parameters']

        cache_size = None
        if "cache_size" in fe_parameters:
            assert self._is_int(fe_parameters['cache_size']), 'Cache size must be a number'
            cache_size = int(fe_parameters['cache_size'])


        pkt_builder = STLPktBuilder(pkt=pkt, vm=STLScVmRaw(instructions, cache_size=cache_size))
        pkt_builder.compile()
        return pkt_builder.get_vm_data()

    def _sanitize_value(self, param_id, val):
        if param_id == "pkt_offset":
            if self._is_int(val):
                return int(val)
            elif val == "Ether.src":
                return 0
            elif val == "Ether.dst":
                return 6
            elif val == "Ether.type":
                return 12
        else:
            if val == "None" or val == "none":
                return None
            if val == "true":
                return True
            elif val == "false":
                return False
            elif re.match("[0-9a-f]{2}([-:])[0-9a-f]{2}(\\1[0-9a-f]{2}){4}$", str(val.lower())):
                return int(str(val).replace(":", ""), 16)

        if self._is_int(val):
            return int(val)

        str_val = str(val)
        return int(str_val, 16) if str_val.startswith("0x") else str_val

    def _get_instruction_parameter_meta(self, param_id):
        for meta in self.instruction_parameter_meta_definitions:
            if meta['id'] == param_id:
                return meta
        raise Scapy_Exception("Unable to get meta for {0}" % param_id)

    def _is_int(self, val):
        try:
            int(val)
            return True
        except ValueError:
            return False

    # @deprecated. to be removed
    def get_all(self,client_v_handler):
        if not (self._verify_version_handler(client_v_handler)):
            raise ScapyException(self._generate_invalid_version_error())
        fields=self._get_all_fields()
        db=self._get_all_db()
        fields_md5 = self._get_md5(fields)
        db_md5 = self._get_md5(db)
        res = {}
        res['db'] = db
        res['fields'] = fields
        res['db_md5'] = db_md5
        res['fields_md5'] = fields_md5
        return res

    def _is_packet_class(self, pkt_class):
        # returns true for final Packet classes. skips aliases and metaclasses
        return issubclass(pkt_class, Packet) and pkt_class.name and pkt_class.fields_desc

    def _getDummyPacket(self, pkt_class):
        if issubclass(pkt_class, Raw):
            # need to have some payload. otherwise won't appear in the binary chunk
            return pkt_class(load=str_to_bytes("dummy"))
        else:
            return pkt_class()


    def _get_payload_classes(self, pkt_class):
        # tries to find, which subclasses allowed.
        # this can take long time, since it tries to build packets with all subclasses(O(N))
        allowed_subclasses = []
        for pkt_subclass in conf.layers:
            if self._is_packet_class(pkt_subclass):
                try:
                    pkt_w_payload = pkt_class() / self._getDummyPacket(pkt_subclass)
                    recreated_pkt = pkt_class(bytes(pkt_w_payload))
                    if type(recreated_pkt.lastlayer()) is pkt_subclass:
                        allowed_subclasses.append(pkt_subclass)
                except Exception as e:
                    # no actions needed on fail, just sliently skip
                    pass
        return allowed_subclasses



    def _get_templates(self):
        templates = []
        for root, subdirs, files in os.walk("templates"):
            for file in files:
                if not file.endswith('.trp'):
                    continue
                try:
                    f = os.path.join(root, file)
                    c = None
                    with open(f, 'r') as templatefile:
                        c = json.loads(templatefile.read())
                    id = f.replace("templates" + os.path.sep, "", 1)
                    id = id.split(os.path.sep)
                    id[-1] = id[-1].replace(".trp", "", 1)
                    id = "/".join(id)
                    t = {
                            "id": id,
                             "meta": {
                                 "name": c["metadata"]["caption"],
                                 "description": ""
                             }
                        }
                    templates.append(t)
                except:
                    pass
        return templates

    def _get_template(self,template):
        id = template["id"]
        f2 = "templates" + os.path.sep + os.path.sep.join(id.split("/")) + ".trp"
        for c in r'[]\;,><&*:%=+@!#^()|?^':
            id = id.replace(c,'')
        id = id.replace("..", "")
        id = id.split("/")
        f = "templates" + os.path.sep + os.path.sep.join(id) + ".trp"
        if f != f2:
            return ""
        with open(f, 'r') as content_file:
            content = base64.b64encode(content_file.read())
        return content


    def _get_fields_definition(self, pkt_class, fieldsDef):
        # fieldsDef - array of field definitions(or empty array)
        fields = []
        for field_desc in pkt_class.fields_desc:
            fieldId = field_desc.name
            field_data = {
                    "id": fieldId,
                    "name": field_desc.name
            }
            for fieldDef in fieldsDef:
                if fieldDef['id'] == fieldId:
                    field_data.update(fieldDef)
            if isinstance(field_desc, EnumField):
                try:
                    field_data["values_dict"] = field_desc.s2i
                    if field_data.get("type") == None:
                        if len(field_data["values_dict"] > 0):
                            field_data["type"] = "ENUM"
                        elif fieldId == 'load':
                            field_data["type"] = "BYTES"
                        else:
                            field_data["type"] = "STRING"
                    field_data["values_dict"] = field_desc.s2i
                except:
                    # MultiEnumField doesn't have s2i. need better handling
                    pass
            fields.append(field_data)
        return fields

    def get_definitions(self,client_v_handler, def_filter):
        # def_filter is an array of classnames or None
        all_classes = Packet.__subclasses__() # as an alternative to conf.layers
        if def_filter:
            all_classes = [c for c in all_classes if c.__name__ in def_filter]
        protocols = []
        for pkt_class in all_classes:
            if self._is_packet_class(pkt_class):
                # enumerate all non-abstract Packet classes
                protocolId = pkt_class.__name__
                protoDef = self.protocol_definitions.get(protocolId) or {}
                protocols.append({
                    "id": protocolId,
                    "name": protoDef.get('name') or pkt_class.name,
                    "fields": self._get_fields_definition(pkt_class, protoDef.get('fields') or [])
                    })
        res = {"protocols": protocols,
               "feInstructionParameters": self.instruction_parameter_meta_definitions,
               "feInstructions": self.field_engine_instructions_meta,
               "feParameters": self.field_engine_parameter_meta_definitions,
               "feTemplates": self.field_engine_templates_definitions}
        return res

    def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
        pkt_class = type(pkt.lastlayer())
        protocolDef = self.protocol_definitions.get(pkt_class.__name__)
        if protocolDef and protocolDef.get('payload'):
            return protocolDef['payload']
        return [c.__name__ for c in self._get_payload_classes(pkt_class)]

    def get_templates(self,client_v_handler):
        return self._get_templates()

    def get_template(self,client_v_handler,template):
        return self._get_template(template)

#input in string encoded base64
    def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
        if not (self._verify_version_handler(client_v_handler)):
            raise ScapyException(self._generate_invalid_version_error())
        fields=self._get_all_fields()
        db=self._get_all_db()
        current_db_md5 = self._get_md5(db)
        current_field_md5 = self._get_md5(fields)
        res = []
        if (field_md5 == current_field_md5):
            if (db_md5 == current_db_md5):
                return True
            else:
                raise ScapyException("Protocol DB is not up to date")
        else:
            raise ScapyException("Fields DB is not up to date")

    def _modify_layer(self, scapy_pkt, scapy_layer, fields):
        for field in fields:
            fieldId = str(field['id'])
            fieldval = self._field_value_from_def(scapy_pkt, scapy_layer, fieldId, field['value'])
            if fieldval is not None:
                scapy_layer.setfieldval(fieldId, fieldval)
            else:
                scapy_layer.delfieldval(fieldId)

    def _is_last_layer(self, layer):
        # can be used, that layer has no payload
        # if true, the layer.payload is likely NoPayload()
        return layer is layer.lastlayer()

#input of binary_pkt must be encoded in base64
    def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
        pkt_bin = b64_to_bytes(binary_pkt)
        scapy_pkt = Ether(pkt_bin)
        if not model_descriptor:
            model_descriptor = []
        for depth in range(len(model_descriptor)):
            model_layer = model_descriptor[depth]
            if model_layer.get('delete') is True:
                # slice packet from the current item
                if depth == 0:
                    scapy_pkt = None
                    break
                else:
                    scapy_pkt[depth-1].payload = None
                    break
            if depth > 0 and self._is_last_layer(scapy_pkt[depth-1]):
                # insert new layer(s) from json definition
                remaining_definitions = model_descriptor[depth:]
                pkt_to_append = self._packet_model_to_scapy_packet(remaining_definitions)
                scapy_pkt = scapy_pkt / pkt_to_append
                break
            # modify fields of existing stack items
            scapy_layer = scapy_pkt[depth]
            if model_layer['id'] != type(scapy_layer).__name__:
                # TODO: support replacing payload, instead of breaking
                raise ScapyException("Protocol id inconsistent")
            if 'fields' in model_layer:
                self._modify_layer(scapy_pkt, scapy_layer, model_layer['fields'])
        return self._pkt_data(scapy_pkt)

    def read_pcap(self,client_v_handler,pcap_base64):
        pcap_bin = b64_to_bytes(pcap_base64)
        pcap = []
        res_packets = []
        with tempfile.NamedTemporaryFile(mode='w+b') as tmpPcap:
            tmpPcap.write(pcap_bin)
            tmpPcap.flush()
            pcap = rdpcap(tmpPcap.name)
        for scapy_packet in pcap:
            res_packets.append(self._pkt_data(scapy_packet))
        return res_packets

    def write_pcap(self,client_v_handler,packets_base64):
        packets = [Ether(b64_to_bytes(pkt_b64)) for pkt_b64 in packets_base64]
        pcap_bin = None
        with tempfile.NamedTemporaryFile(mode='r+b') as tmpPcap:
            wrpcap(tmpPcap.name, packets)
            pcap_bin = tmpPcap.read()
        return bytes_to_b64(pcap_bin)


#---------------------------------------------------------------------------