SNAT: IPFIX logging (VPP-445)

Change-Id: I8450217dd43a1cd9f510e40dfb22274ffc33a4c6
Signed-off-by: Matus Fabian <matfabia@cisco.com>

1 files changed, 539 insertions, 0 deletions

diff --git a/test/ipfix.py b/test/ipfix.py
new file mode 100644
index 00000000000..deaff67ba42
--- /dev/null
+++ b/test/ipfix.py
@@ -0,0 +1,539 @@
+#!/usr/bin/env python
+# IPFIX support for Scapy (RFC7011)
+
+from scapy.all import *
+
+
+# IPFIX Information Elements http://www.iana.org/assignments/ipfix/ipfix.xhtml
+information_elements = {
+    1:   "octetDeltaCount",
+    2:   "packetDeltaCount",
+    3:   "deltaFlowCount",
+    4:   "protocolIdentifier",
+    5:   "ipClassOfService",
+    6:   "tcpControlBits",
+    7:   "sourceTransportPort",
+    8:   "sourceIPv4Address",
+    9:   "sourceIPv4PrefixLength",
+    10:  "ingressInterface",
+    11:  "destinationTransportPort",
+    12:  "destinationIPv4Address",
+    13:  "destinationIPv4PrefixLength",
+    14:  "egressInterface",
+    15:  "ipNextHopIPv4Address",
+    16:  "bgpSourceAsNumber",
+    17:  "bgpDestinationAsNumber",
+    18:  "bgpNextHopIPv4Address",
+    19:  "postMCastPacketDeltaCount",
+    20:  "postMCastOctetDeltaCount",
+    21:  "flowEndSysUpTime",
+    22:  "flowStartSysUpTime",
+    23:  "postOctetDeltaCount",
+    24:  "postPacketDeltaCount",
+    25:  "minimumIpTotalLength",
+    26:  "maximumIpTotalLength",
+    27:  "sourceIPv6Address",
+    28:  "destinationIPv6Address",
+    29:  "sourceIPv6PrefixLength",
+    30:  "destinationIPv6PrefixLength",
+    31:  "flowLabelIPv6",
+    32:  "icmpTypeCodeIPv4",
+    33:  "igmpType",
+    34:  "samplingInterval",
+    35:  "samplingAlgorithm",
+    36:  "flowActiveTimeout",
+    37:  "flowIdleTimeout",
+    38:  "engineType",
+    39:  "engineId",
+    40:  "exportedOctetTotalCount",
+    41:  "exportedMessageTotalCount",
+    42:  "exportedFlowRecordTotalCount",
+    43:  "ipv4RouterSc",
+    44:  "sourceIPv4Prefix",
+    45:  "destinationIPv4Prefix",
+    46:  "mplsTopLabelType",
+    47:  "mplsTopLabelIPv4Address",
+    48:  "samplerId",
+    49:  "samplerMode",
+    50:  "samplerRandomInterval",
+    51:  "classId",
+    52:  "minimumTTL",
+    53:  "maximumTTL",
+    54:  "fragmentIdentification",
+    55:  "postIpClassOfService",
+    56:  "sourceMacAddress",
+    57:  "postDestinationMacAddress",
+    58:  "vlanId",
+    59:  "postVlanId",
+    60:  "ipVersion",
+    61:  "flowDirection",
+    62:  "ipNextHopIPv6Address",
+    63:  "bgpNextHopIPv6Address",
+    64:  "ipv6ExtensionHeaders",
+    70:  "mplsTopLabelStackSection",
+    71:  "mplsLabelStackSection2",
+    72:  "mplsLabelStackSection3",
+    73:  "mplsLabelStackSection4",
+    74:  "mplsLabelStackSection5",
+    75:  "mplsLabelStackSection6",
+    76:  "mplsLabelStackSection7",
+    77:  "mplsLabelStackSection8",
+    78:  "mplsLabelStackSection9",
+    79:  "mplsLabelStackSection10",
+    80:  "destinationMacAddress",
+    81:  "postSourceMacAddress",
+    82:  "interfaceName",
+    83:  "interfaceDescription",
+    84:  "samplerName",
+    85:  "octetTotalCount",
+    86:  "packetTotalCount",
+    87:  "flagsAndSamplerId",
+    88:  "fragmentOffset",
+    89:  "forwardingStatus",
+    90:  "mplsVpnRouteDistinguisher",
+    91:  "mplsTopLabelPrefixLength",
+    92:  "srcTrafficIndex",
+    93:  "dstTrafficIndex",
+    94:  "applicationDescription",
+    95:  "applicationId",
+    96:  "applicationName",
+    98:  "postIpDiffServCodePoint",
+    99:  "multicastReplicationFactor",
+    100: "className",
+    101: "classificationEngineId",
+    102: "layer2packetSectionOffset",
+    103: "layer2packetSectionSize",
+    104: "layer2packetSectionData",
+    128: "bgpNextAdjacentAsNumber",
+    129: "bgpPrevAdjacentAsNumber",
+    130: "exporterIPv4Address",
+    131: "exporterIPv6Address",
+    132: "droppedOctetDeltaCount",
+    133: "droppedPacketDeltaCount",
+    134: "droppedOctetTotalCount",
+    135: "droppedPacketTotalCount",
+    136: "flowEndReason",
+    137: "commonPropertiesId",
+    138: "observationPointId",
+    139: "icmpTypeCodeIPv6",
+    140: "mplsTopLabelIPv6Address",
+    141: "lineCardId",
+    142: "portId",
+    143: "meteringProcessId",
+    144: "exportingProcessId",
+    145: "templateId",
+    146: "wlanChannelId",
+    147: "wlanSSID",
+    148: "flowId",
+    149: "observationDomainId",
+    150: "flowStartSeconds",
+    151: "flowEndSeconds",
+    152: "flowStartMilliseconds",
+    153: "flowEndMilliseconds",
+    154: "flowStartMicroseconds",
+    155: "flowEndMicroseconds",
+    156: "flowStartNanoseconds",
+    157: "flowEndNanoseconds",
+    158: "flowStartDeltaMicroseconds",
+    159: "flowEndDeltaMicroseconds",
+    160: "systemInitTimeMilliseconds",
+    161: "flowDurationMilliseconds",
+    162: "flowDurationMicroseconds",
+    163: "observedFlowTotalCount",
+    164: "ignoredPacketTotalCount",
+    165: "ignoredOctetTotalCount",
+    166: "notSentFlowTotalCount",
+    167: "notSentPacketTotalCount",
+    168: "notSentOctetTotalCount",
+    169: "destinationIPv6Prefix",
+    170: "sourceIPv6Prefix",
+    171: "postOctetTotalCount",
+    172: "postPacketTotalCount",
+    173: "flowKeyIndicator",
+    174: "postMCastPacketTotalCount",
+    175: "postMCastOctetTotalCount",
+    176: "icmpTypeIPv4",
+    177: "icmpCodeIPv4",
+    178: "icmpTypeIPv6",
+    179: "icmpCodeIPv6",
+    180: "udpSourcePort",
+    181: "udpDestinationPort",
+    182: "tcpSourcePort",
+    183: "tcpDestinationPort",
+    184: "tcpSequenceNumber",
+    185: "tcpAcknowledgementNumber",
+    186: "tcpWindowSize",
+    187: "tcpUrgentPointer",
+    188: "tcpHeaderLength",
+    189: "ipHeaderLength",
+    190: "totalLengthIPv4",
+    191: "payloadLengthIPv6",
+    192: "ipTTL",
+    193: "nextHeaderIPv6",
+    194: "mplsPayloadLength",
+    195: "ipDiffServCodePoint",
+    196: "ipPrecedence",
+    197: "fragmentFlags",
+    198: "octetDeltaSumOfSquares",
+    199: "octetTotalSumOfSquares",
+    200: "mplsTopLabelTTL",
+    201: "mplsLabelStackLength",
+    202: "mplsLabelStackDepth",
+    203: "mplsTopLabelExp",
+    204: "ipPayloadLength",
+    205: "udpMessageLength",
+    206: "isMulticast",
+    207: "ipv4IHL",
+    208: "ipv4Options",
+    209: "tcpOptions",
+    210: "paddingOctets",
+    211: "collectorIPv4Address",
+    212: "collectorIPv6Address",
+    213: "exportInterface",
+    214: "exportProtocolVersion",
+    215: "exportTransportProtocol",
+    216: "collectorTransportPort",
+    217: "exporterTransportPort",
+    218: "tcpSynTotalCount",
+    219: "tcpFinTotalCount",
+    220: "tcpRstTotalCount",
+    221: "tcpPshTotalCount",
+    222: "tcpAckTotalCount",
+    223: "tcpUrgTotalCount",
+    224: "ipTotalLength",
+    225: "postNATSourceIPv4Address",
+    226: "postNATDestinationIPv4Address",
+    227: "postNAPTSourceTransportPort",
+    228: "postNAPTDestinationTransportPort",
+    229: "natOriginatingAddressRealm",
+    230: "natEvent",
+    231: "initiatorOctets",
+    232: "responderOctets",
+    233: "firewallEvent",
+    234: "ingressVRFID",
+    235: "egressVRFID",
+    236: "VRFname",
+    237: "postMplsTopLabelExp",
+    238: "tcpWindowScale",
+    239: "biflowDirection",
+    240: "ethernetHeaderLength",
+    241: "ethernetPayloadLength",
+    242: "ethernetTotalLength",
+    243: "dot1qVlanId",
+    244: "dot1qPriority",
+    245: "dot1qCustomerVlanId",
+    246: "dot1qCustomerPriority",
+    247: "metroEvcId",
+    248: "metroEvcType",
+    249: "pseudoWireId",
+    250: "pseudoWireType",
+    251: "pseudoWireControlWord",
+    252: "ingressPhysicalInterface",
+    253: "egressPhysicalInterface",
+    254: "postDot1qVlanId",
+    255: "postDot1qCustomerVlanId",
+    256: "ethernetType",
+    257: "postIpPrecedence",
+    258: "collectionTimeMilliseconds",
+    259: "exportSctpStreamId",
+    260: "maxExportSeconds",
+    261: "maxFlowEndSeconds",
+    262: "messageMD5Checksum",
+    263: "messageScope",
+    264: "minExportSeconds",
+    265: "minFlowStartSeconds",
+    266: "opaqueOctets",
+    267: "sessionScope",
+    268: "maxFlowEndMicroseconds",
+    269: "maxFlowEndMilliseconds",
+    270: "maxFlowEndNanoseconds",
+    271: "minFlowStartMicroseconds",
+    272: "minFlowStartMilliseconds",
+    273: "minFlowStartNanoseconds",
+    274: "collectorCertificate",
+    275: "exporterCertificate",
+    276: "dataRecordsReliability",
+    277: "observationPointType",
+    278: "newConnectionDeltaCount",
+    279: "connectionSumDurationSeconds",
+    280: "connectionTransactionId",
+    281: "postNATSourceIPv6Address",
+    282: "postNATDestinationIPv6Address",
+    283: "natPoolId",
+    284: "natPoolName",
+    285: "anonymizationFlags",
+    286: "anonymizationTechnique",
+    287: "informationElementIndex",
+    288: "p2pTechnology",
+    289: "tunnelTechnology",
+    290: "encryptedTechnology",
+    291: "basicList",
+    292: "subTemplateList",
+    293: "subTemplateMultiList",
+    294: "bgpValidityState",
+    295: "IPSecSPI",
+    296: "greKey",
+    297: "natType",
+    298: "initiatorPackets",
+    299: "responderPackets",
+    300: "observationDomainName",
+    301: "selectionSequenceId",
+    302: "selectorId",
+    303: "informationElementId",
+    304: "selectorAlgorithm",
+    305: "samplingPacketInterval",
+    306: "samplingPacketSpace",
+    307: "samplingTimeInterval",
+    308: "samplingTimeSpace",
+    309: "samplingSize",
+    310: "samplingPopulation",
+    311: "samplingProbability",
+    312: "dataLinkFrameSize",
+    313: "ipHeaderPacketSection",
+    314: "ipPayloadPacketSection",
+    315: "dataLinkFrameSection",
+    316: "mplsLabelStackSection",
+    317: "mplsPayloadPacketSection",
+    318: "selectorIdTotalPktsObserved",
+    319: "selectorIdTotalPktsSelected",
+    320: "absoluteError",
+    321: "relativeError",
+    322: "observationTimeSeconds",
+    323: "observationTimeMilliseconds",
+    324: "observationTimeMicroseconds",
+    325: "observationTimeNanoseconds",
+    326: "digestHashValue",
+    327: "hashIPPayloadOffset",
+    328: "hashIPPayloadSize",
+    329: "hashOutputRangeMin",
+    330: "hashOutputRangeMax",
+    331: "hashSelectedRangeMin",
+    332: "hashSelectedRangeMax",
+    333: "hashDigestOutput",
+    334: "hashInitialiserValue",
+    335: "selectorName",
+    336: "upperCILimit",
+    337: "lowerCILimit",
+    338: "confidenceLevel",
+    339: "informationElementDataType",
+    340: "informationElementDescription",
+    341: "informationElementName",
+    342: "informationElementRangeBegin",
+    343: "informationElementRangeEnd",
+    344: "informationElementSemantics",
+    345: "informationElementUnits",
+    346: "privateEnterpriseNumber",
+    347: "virtualStationInterfaceId",
+    348: "virtualStationInterfaceName",
+    349: "virtualStationUUID",
+    350: "virtualStationName",
+    351: "layer2SegmentId",
+    352: "layer2OctetDeltaCount",
+    353: "layer2OctetTotalCount",
+    354: "ingressUnicastPacketTotalCount",
+    355: "ingressMulticastPacketTotalCount",
+    356: "ingressBroadcastPacketTotalCount",
+    357: "egressUnicastPacketTotalCount",
+    358: "egressBroadcastPacketTotalCount",
+    359: "monitoringIntervalStartMilliSeconds",
+    360: "monitoringIntervalEndMilliSeconds",
+    361: "portRangeStart",
+    362: "portRangeEnd",
+    363: "portRangeStepSize",
+    364: "portRangeNumPorts",
+    365: "staMacAddress",
+    366: "staIPv4Address",
+    367: "wtpMacAddress",
+    368: "ingressInterfaceType",
+    369: "egressInterfaceType",
+    370: "rtpSequenceNumber",
+    371: "userName",
+    372: "applicationCategoryName",
+    373: "applicationSubCategoryName",
+    374: "applicationGroupName",
+    375: "originalFlowsPresent",
+    376: "originalFlowsInitiated",
+    377: "originalFlowsCompleted",
+    378: "distinctCountOfSourceIPAddress",
+    379: "distinctCountOfDestinationIPAddress",
+    380: "distinctCountOfSourceIPv4Address",
+    381: "distinctCountOfDestinationIPv4Address",
+    382: "distinctCountOfSourceIPv6Address",
+    383: "distinctCountOfDestinationIPv6Address",
+    384: "valueDistributionMethod",
+    385: "rfc3550JitterMilliseconds",
+    386: "rfc3550JitterMicroseconds",
+    387: "rfc3550JitterNanoseconds",
+    388: "dot1qDEI",
+    389: "dot1qCustomerDEI",
+    390: "flowSelectorAlgorithm",
+    391: "flowSelectedOctetDeltaCount",
+    392: "flowSelectedPacketDeltaCount",
+    393: "flowSelectedFlowDeltaCount",
+    394: "selectorIDTotalFlowsObserved",
+    395: "selectorIDTotalFlowsSelected",
+    396: "samplingFlowInterval",
+    397: "samplingFlowSpacing",
+    398: "flowSamplingTimeInterval",
+    399: "flowSamplingTimeSpacing",
+    400: "hashFlowDomain",
+    401: "transportOctetDeltaCount",
+    402: "transportPacketDeltaCount",
+    403: "originalExporterIPv4Address",
+    404: "originalExporterIPv6Address",
+    405: "originalObservationDomainId",
+    406: "intermediateProcessId",
+    407: "ignoredDataRecordTotalCount",
+    408: "dataLinkFrameType",
+    409: "sectionOffset",
+    410: "sectionExportedOctets",
+    411: "dot1qServiceInstanceTag",
+    412: "dot1qServiceInstanceId",
+    413: "dot1qServiceInstancePriority",
+    414: "dot1qCustomerSourceMacAddress",
+    415: "dot1qCustomerDestinationMacAddress",
+    417: "postLayer2OctetDeltaCount",
+    418: "postMCastLayer2OctetDeltaCount",
+    420: "postLayer2OctetTotalCount",
+    421: "postMCastLayer2OctetTotalCount",
+    422: "minimumLayer2TotalLength",
+    423: "maximumLayer2TotalLength",
+    424: "droppedLayer2OctetDeltaCount",
+    425: "droppedLayer2OctetTotalCount",
+    426: "ignoredLayer2OctetTotalCount",
+    427: "notSentLayer2OctetTotalCount",
+    428: "layer2OctetDeltaSumOfSquares",
+    429: "layer2OctetTotalSumOfSquares",
+    430: "layer2FrameDeltaCount",
+    431: "layer2FrameTotalCount",
+    432: "pseudoWireDestinationIPv4Address",
+    433: "ignoredLayer2FrameTotalCount",
+    434: "mibObjectValueInteger",
+    435: "mibObjectValueOctetString",
+    436: "mibObjectValueOID",
+    437: "mibObjectValueBits",
+    438: "mibObjectValueIPAddress",
+    439: "mibObjectValueCounter",
+    440: "mibObjectValueGauge",
+    441: "mibObjectValueTimeTicks",
+    442: "mibObjectValueUnsigned",
+    443: "mibObjectValueTable",
+    444: "mibObjectValueRow",
+    445: "mibObjectIdentifier",
+    446: "mibSubIdentifier",
+    447: "mibIndexIndicator",
+    448: "mibCaptureTimeSemantics",
+    449: "mibContextEngineID",
+    450: "mibContextName",
+    451: "mibObjectName",
+    452: "mibObjectDescription",
+    453: "mibObjectSyntax",
+    454: "mibModuleName",
+    455: "mobileIMSI",
+    456: "mobileMSISDN",
+    457: "httpStatusCode",
+    458: "sourceTransportPortsLimit",
+    459: "httpRequestMethod",
+    460: "httpRequestHost",
+    461: "httpRequestTarget",
+    462: "httpMessageVersion"
+}
+
+
+class IPFIX(Packet):
+    name = "IPFIX"
+    fields_desc = [ShortField("version", 10),
+                   ShortField("length", None),
+                   IntField("exportTime", None),
+                   IntField("sequenceNumber", 1),
+                   IntField("observationDomainID", 1)]
+
+
+class FieldSpecifier(Packet):
+    name = "Field Specifier"
+    fields_desc = [ShortEnumField(
+        "informationElement", None, information_elements),
+        ShortField("fieldLength", None)]
+
+    def extract_padding(self, s):
+        return "", s
+
+
+class Template(Packet):
+    name = "Template"
+    fields_desc = [ShortField("templateID", 256),
+                   FieldLenField("fieldCount", None, count_of="fields"),
+                   PacketListField("templateFields", [], FieldSpecifier,
+                                   count_from=lambda p: p.fieldCount)]
+
+
+class Data(Packet):
+    name = "Data"
+    fields_desc = [
+        StrLenField("data", "", length_from=lambda p: p.underlayer.length - 4)]
+
+    def extract_padding(self, s):
+        return "", s
+
+
+class Set(Packet):
+    name = "Set"
+    fields_desc = [ShortField("setID", 256),
+                   ShortField("length", None)]
+
+    def guess_payload_class(self, payload):
+        if self.setID == 2:
+            return Template
+        elif self.setID > 255:
+            return Data
+        else:
+            return Packet.guess_payload_class(self, payload)
+
+
+bind_layers(IPFIX, Set)
+bind_layers(UDP, IPFIX, dport=4739)
+
+
+class IPFIXDecoder(object):
+    """ IPFIX data set decoder """
+
+    def __init__(self):
+        self._templates = []
+
+    def add_template(self, template):
+        """
+        Add IPFIX tempalte
+
+        :param template: IPFIX template
+        """
+        templateID = template.templateID
+        fields = []
+        rec_len = 0
+        for field in template.templateFields:
+            fields.append(
+                {'name': field.informationElement, 'len': field.fieldLength})
+            rec_len += field.fieldLength
+        self._templates.append(
+            {'id': templateID, 'fields': fields, 'rec_len': rec_len})
+
+    def decode_data_set(self, data_set):
+        """
+        Decode IPFIX data
+
+        :param data_set: IPFIX data set
+        :returns: List of decoded data records.
+        """
+        data = []
+        for template in self._templates:
+            if template['id'] == data_set.setID:
+                offset = 0
+                d = data_set[Data].data
+                for i in range(len(d) / template['rec_len']):
+                    record = {}
+                    for field in template['fields']:
+                        f = d[offset:offset + field['len']]
+                        offset += field['len']
+                        record.update({field['name']: f})
+                    data.append(record)
+                break
+        return data