#!/usr/bin/env python3
# IPFIX support for Scapy (RFC7011)
from scapy.all import (
bind_layers,
FieldLenField,
IntField,
Packet,
PacketListField,
ShortEnumField,
ShortField,
StrLenField,
)
from scapy.layers.inet import UDP
# IPFIX Information Elements http://www.iana.org/assignments/ipfix/ipfix.xhtml
information_elements = {
1: "octetDeltaCount",
2: "packetDeltaCount",
3: "deltaFlowCount",
4: "protocolIdentifier",
5: "ipClassOfService",
6: "tcpControlBits",
7: "sourceTransportPort",
8: "sourceIPv4Address",
9: "sourceIPv4PrefixLength",
10: "ingressInterface",
11: "destinationTransportPort",
12: "destinationIPv4Address",
13: "destinationIPv4PrefixLength",
14: "egressInterface",
15: "ipNextHopIPv4Address",
16: "bgpSourceAsNumber",
17: "bgpDestinationAsNumber",
18: "bgpNextHopIPv4Address",
19: "postMCastPacketDeltaCount",
20: "postMCastOctetDeltaCount",
21: "flowEndSysUpTime",
22: "flowStartSysUpTime",
23: "postOctetDeltaCount",
24: "postPacketDeltaCount",
25: "minimumIpTotalLength",
26: "maximumIpTotalLength",
27: "sourceIPv6Address",
28: "destinationIPv6Address",
29: "sourceIPv6PrefixLength",
30: "destinationIPv6PrefixLength",
31: "flowLabelIPv6",
32: "icmpTypeCodeIPv4",
33: "igmpType",
34: "samplingInterval",
35: "samplingAlgorithm",
36: "flowActiveTimeout",
37: "flowIdleTimeout",
38: "engineType",
39: "engineId",
40: "exportedOctetTotalCount",
41: "exportedMessageTotalCount",
42: "exportedFlowRecordTotalCount",
43: "ipv4RouterSc",
44: "sourceIPv4Prefix",
45: "destinationIPv4Prefix",
46: "mplsTopLabelType",
47: "mplsTopLabelIPv4Address",
48: "samplerId",
49: "samplerMode",
50: "samplerRandomInterval",
51: "classId",
52: "minimumTTL",
53: "maximumTTL",
54: "fragmentIdentification",
55: "postIpClassOfService",
56: "sourceMacAddress",
57: "postDestinationMacAddress",
58: "vlanId",
59: "postVlanId",
60: "ipVersion",
61: "flowDirection",
62: "ipNextHopIPv6Address",
63: "bgpNextHopIPv6Address",
64: "ipv6ExtensionHeaders",
70: "mplsTopLabelStackSection",
71: "mplsLabelStackSection2",
72: "mplsLabelStackSection3",
73: "mplsLabelStackSection4",
74: "mplsLabelStackSection5",
75: "mplsLabelStackSection6",
76: "mplsLabelStackSection7",
77: "mplsLabelStackSection8",
78: "mplsLabelStackSection9",
79: "mplsLabelStackSection10",
80: "destinationMacAddress",
81: "postSourceMacAddress",
82: "interfaceName",
83: "interfaceDescription",
84: "samplerName",
85: "octetTotalCount",
86: "packetTotalCount",
87: "flagsAndSamplerId",
88: "fragmentOffset",
89: "forwardingStatus",
90: "mplsVpnRouteDistinguisher",
91: "mplsTopLabelPrefixLength",
92: "srcTrafficIndex",
93: "dstTrafficIndex",
94: "applicationDescription",
95: "applicationId",
96: "applicationName",
98: "postIpDiffServCodePoint",
99: "multicastReplicationFactor",
100: "className",
101: "classificationEngineId",
102: "layer2packetSectionOffset",
103: "layer2packetSectionSize",
104: "layer2packetSectionData",
128: "bgpNextAdjacentAsNumber",
129: "bgpPrevAdjacentAsNumber",
130: "exporterIPv4Address",
131: "exporterIPv6Address",
132: "droppedOctetDeltaCount",
133: "droppedPacketDeltaCount",
134: "droppedOctetTotalCount",
135: "droppedPacketTotalCount",
136: "flowEndReason",
137: "commonPropertiesId",
138: "observationPointId",
139: "icmpTypeCodeIPv6",
140: "mplsTopLabelIPv6Address",
141: "lineCardId",
142: "portId",
143: "meteringProcessId",
144: "exportingProcessId",
145: "templateId",
146: "wlanChannelId",
147: "wlanSSID",
148: "flowId",
149: "observationDomainId",
150: "flowStartSeconds",
151: "flowEndSeconds",
152: "flowStartMilliseconds",
153: "flowEndMilliseconds",
154: "flowStartMicroseconds",
155: "flowEndMicroseconds",
156: "flowStartNanoseconds",
157: "flowEndNanoseconds",
158: "flowStartDeltaMicroseconds",
159: "flowEndDeltaMicroseconds",
160: "systemInitTimeMilliseconds",
161: "flowDurationMilliseconds",
162: "flowDurationMicroseconds",
163: "observedFlowTotalCount",
164: "ignoredPacketTotalCount",
165: "ignoredOctetTotalCount",
166: "notSentFlowTotalCount",
167: "notSentPacketTotalCount",
168: "notSentOctetTotalCount",
169: "destinationIPv6Prefix",
170: "sourceIPv6Prefix",
171: "postOctetTotalCount",
172: "postPacketTotalCount",
173: "flowKeyIndicator",
174: "postMCastPacketTotalCount",
175: "postMCastOctetTotalCount",
176: "icmpTypeIPv4",
177: "icmpCodeIPv4",
178: "icmpTypeIPv6",
179: "icmpCodeIPv6",
180: "udpSourcePort",
181: "udpDestinationPort",
182: "tcpSourcePort",
183: "tcpDestinationPort",
184: "tcpSequenceNumber",
185: "tcpAcknowledgementNumber",
186: "tcpWindowSize",
187: "tcpUrgentPointer",
188: "tcpHeaderLength",
189: "ipHeaderLength",
190: "totalLengthIPv4",
191: "payloadLengthIPv6",
192: "ipTTL",
193: "nextHeaderIPv6",
194: "mplsPayloadLength",
195: "ipDiffServCodePoint",
196: "ipPrecedence",
197: "fragmentFlags",
198: "octetDeltaSumOfSquares",
199: "octetTotalSumOfSquares",
200: "mplsTopLabelTTL",
201: "mplsLabelStackLength",
202: "mplsLabelStackDepth",
203: "mplsTopLabelExp",
204: "ipPayloadLength",
205: "udpMessageLength",
206: "isMulticast",
207: "ipv4IHL",
208: "ipv4Options",
209: "tcpOptions",
210: "paddingOctets",
211: "collectorIPv4Address",
212: "collectorIPv6Address",
213: "exportInterface",
214: "exportProtocolVersion",
215: "exportTransportProtocol",
216: "collectorTransportPort",
217: "exporterTransportPort",
218: "tcpSynTotalCount",
219: "tcpFinTotalCount",
220: "tcpRstTotalCount",
221: "tcpPshTotalCount",
222: "tcpAckTotalCount",
223: "tcpUrgTotalCount",
224: "ipTotalLength",
225: "postNATSourceIPv4Address",
226: "postNATDestinationIPv4Address",
227: "postNAPTSourceTransportPort",
228: "postNAPTDestinationTransportPort",
229: "natOriginatingAddressRealm",
230: "natEvent",
231: "initiatorOctets",
232: "responderOctets",
233: "firewallEvent",
234: "ingressVRFID",
235: "egressVRFID",
236: "VRFname",
237: "postMplsTopLabelExp",
238: "tcpWindowScale",
239: "biflowDirection",
240: "ethernetHeaderLength",
241: "ethernetPayloadLength",
242: "ethernetTotalLength",
243: "dot1qVlanId",
244: "dot1qPriority",
245: "dot1qCustomerVlanId",
246: "dot1qCustomerPriority",
247: "metroEvcId",
248: "metroEvcType",
249: "pseudoWireId",
250: "pseudoWireType",
251: "pseudoWireControlWord",
252: "ingressPhysicalInterface",
253: "egressPhysicalInterface",
254: "postDot1qVlanId",
255: "postDot1qCustomerVlanId",
256: "ethernetType",
257: "postIpPrecedence",
258: "collectionTimeMilliseconds",
259: "exportSctpStreamId",
260: "maxExportSeconds",
261: "maxFlowEndSeconds",
262: "messageMD5Checksum",
263: "messageScope",
264: "minExportSeconds",
265: "minFlowStartSeconds",
266: "opaqueOctets",
267: "sessionScope",
268: "maxFlowEndMicroseconds",
269: "maxFlowEndMilliseconds",
270: "maxFlowEndNanoseconds",
271: "minFlowStartMicroseconds",
272: "minFlowStartMilliseconds",
273: "minFlowStartNanoseconds",
274: "collectorCertificate",
275: "exporterCertificate",
276: "dataRecordsReliability",
277: "observationPointType",
278: "newConnectionDeltaCount",
279: "connectionSumDurationSeconds",
280: "connectionTransactionId",
281: "postNATSourceIPv6Address",
282: "postNATDestinationIPv6Address",
283: "natPoolId",
284: "natPoolName",
285: "anonymizationFlags",
286: "anonymizationTechnique",
287: "informationElementIndex",
288: "p2pTechnology",
289: "tunnelTechnology",
290: "encryptedTechnology",
291: "basicList",
292: "subTemplateList",
293: "subTemplateMultiList",
294: "bgpValidityState",
295: "IPSecSPI",
296: "greKey",
297: "natType",
298: "initiatorPackets",
299: "responderPackets",
300: "observationDomainName",
301: "selectionSequenceId",
302: "selectorId",
303: "informationElementId",
304: "selectorAlgorithm",
305: "samplingPacketInterval",
306: "samplingPacketSpace",
307: "samplingTimeInterval",
308: "samplingTimeSpace",
309: "samplingSize",
310: "samplingPopulation",
311: "samplingProbability",
312: "dataLinkFrameSize",
313: "ipHeaderPacketSection",
314: "ipPayloadPacketSection",
315: "dataLinkFrameSection",
316: "mplsLabelStackSection",
317: "mplsPayloadPacketSection",
318: "selectorIdTotalPktsObserved",
319: "selectorIdTotalPktsSelected",
320: "absoluteError",
321: "relativeError",
322: "observationTimeSeconds",
323: "observationTimeMilliseconds",
324: "observationTimeMicroseconds",
325: "observationTimeNanoseconds",
326: "digestHashValue",
327: "hashIPPayloadOffset",
328: "hashIPPayloadSize",
329: "hashOutputRangeMin",
330: "hashOutputRangeMax",
331: "hashSelectedRangeMin",
332: "hashSelectedRangeMax",
333: "hashDigestOutput",
334: "hashInitialiserValue",
335: "selectorName",
336: "upperCILimit",
337: "lowerCILimit",
338: "confidenceLevel",
339: "informationElementDataType",
340: "informationElementDescription",
341: "informationElementName",
342: "informationElementRangeBegin",
343: "informationElementRangeEnd",
344: "informationElementSemantics",
345: "informationElementUnits",
346: "privateEnterpriseNumber",
347: "virtualStationInterfaceId",
348: "virtualStationInterfaceName",
349: "virtualStationUUID",
350: "virtualStationName",
351: "layer2SegmentId",
352: "layer2OctetDeltaCount",
353: "layer2OctetTotalCount",
354: "ingressUnicastPacketTotalCount",
355: "ingressMulticastPacketTotalCount",
356: "ingressBroadcastPacketTotalCount",
357: "egressUnicastPacketTotalCount",
358: "egressBroadcastPacketTotalCount",
359: "monitoringIntervalStartMilliSeconds",
360: "monitoringIntervalEndMilliSeconds",
361: "portRangeStart",
362: "portRangeEnd",
363: "portRangeStepSize",
364: "portRangeNumPorts",
365: "staMacAddress",
366: "staIPv4Address",
367: "wtpMacAddress",
368: "ingressInterfaceType",
369: "egressInterfaceType",
370: "rtpSequenceNumber",
371: "userName",
372: "applicationCategoryName",
373: "applicationSubCategoryName",
374: "applicationGroupName",
375: "originalFlowsPresent",
376: "originalFlowsInitiated",
377: "originalFlowsCompleted",
378: "distinctCountOfSourceIPAddress",
379: "distinctCountOfDestinationIPAddress",
380: "distinctCountOfSourceIPv4Address",
381: "distinctCountOfDestinationIPv4Address",
382: "distinctCountOfSourceIPv6Address",
383: "distinctCountOfDestinationIPv6Address",
384: "valueDistributionMethod",
385: "rfc3550JitterMilliseconds",
386: "rfc3550JitterMicroseconds",
387: "rfc3550JitterNanoseconds",
388: "dot1qDEI",
389: "dot1qCustomerDEI",
390: "flowSelectorAlgorithm",
391: "flowSelectedOctetDeltaCount",
392: "flowSelectedPacketDeltaCount",
393: "flowSelectedFlowDeltaCount",
394: "selectorIDTotalFlowsObserved",
395: "selectorIDTotalFlowsSelected",
396: "samplingFlowInterval",
397: "samplingFlowSpacing",
398: "flowSamplingTimeInterval",
399: "flowSamplingTimeSpacing",
400: "hashFlowDomain",
401: "transportOctetDeltaCount",
402: "transportPacketDeltaCount",
403: "originalExporterIPv4Address",
404: "originalExporterIPv6Address",
405: "originalObservationDomainId",
406: "intermediateProcessId",
407: "ignoredDataRecordTotalCount",
408: "dataLinkFrameType",
409: "sectionOffset",
410: "sectionExportedOctets",
411: "dot1qServiceInstanceTag",
412: "dot1qServiceInstanceId",
413: "dot1qServiceInstancePriority",
414: "dot1qCustomerSourceMacAddress",
415: "dot1qCustomerDestinationMacAddress",
417: "postLayer2OctetDeltaCount",
418: "postMCastLayer2OctetDeltaCount",
420: "postLayer2OctetTotalCount",
421: "postMCastLayer2OctetTotalCount",
422: "minimumLayer2TotalLength",
423: "maximumLayer2TotalLength",
424: "droppedLayer2OctetDeltaCount",
425: "droppedLayer2OctetTotalCount",
426: "ignoredLayer2OctetTotalCount",
427: "notSentLayer2OctetTotalCount",
428: "layer2OctetDeltaSumOfSquares",
429: "layer2OctetTotalSumOfSquares",
430: "layer2FrameDeltaCount",
431: "layer2FrameTotalCount",
432: "pseudoWireDestinationIPv4Address",
433: "ignoredLayer2FrameTotalCount",
434: "mibObjectValueInteger",
435: "mibObjectValueOctetString",
436: "mibObjectValueOID",
437: "mibObjectValueBits",
438: "mibObjectValueIPAddress",
439: "mibObjectValueCounter",
440: "mibObjectValueGauge",
441: "mibObjectValueTimeTicks",
442: "mibObjectValueUnsigned",
443: "mibObjectValueTable",
444: "mibObjectValueRow",
445: "mibObjectIdentifier",
446: "mibSubIdentifier",
447: "mibIndexIndicator",
448: "mibCaptureTimeSemantics",
449: "mibContextEngineID",
450: "mibContextName",
451: "mibObjectName",
452: "mibObjectDescription",
453: "mibObjectSyntax",
454: "mibModuleName",
455: "mobileIMSI",
456: "mobileMSISDN",
457: "httpStatusCode",
458: "sourceTransportPortsLimit",
459: "httpRequestMethod",
460: "httpRequestHost",
461: "httpRequestTarget",
462: "httpMessageVersion",
466: "natQuotaExceededEvent",
471: "maxSessionEntries",
472: "maxBIBEntries",
473: "maxEntriesPerUser",
475: "maxFragmentsPendingReassembly",
}
class IPFIX(Packet):
name = "IPFIX"
fields_desc = [
ShortField("version", 10),
ShortField("length", None),
IntField("exportTime", None),
IntField("sequenceNumber", 1),
IntField("observationDomainID", 1),
]
class FieldSpecifier(Packet):
name = "Field Specifier"
fields_desc = [
ShortEnumField("informationElement", None, information_elements),
ShortField("fieldLength", None),
]
def extract_padding(self, s):
return "", s
class Template(Packet):
name = "Template"
fields_desc = [
ShortField("templateID", 256),
FieldLenField("fieldCount", None, count_of="fields"),
PacketListField(
"templateFields", [], FieldSpecifier, count_from=lambda p: p.fieldCount
),
]
class Data(Packet):
name = "Data"
fields_desc = [
StrLenField("data", "", length_from=lambda p: p.underlayer.length - 4)
]
def extract_padding(self, s):
return "", s
class Set(Packet):
name = "Set"
fields_desc = [ShortField("setID", 256), ShortField("length", None)]
def guess_payload_class(self, payload):
if self.setID == 2:
return Template
elif self.setID > 255:
return Data
else:
return Packet.guess_payload_class(self, payload)
bind_layers(IPFIX, Set)
bind_layers(UDP, IPFIX, dport=4739)
class IPFIXDecoder:
"""IPFIX data set decoder"""
def __init__(self):
self._templates = []
def add_template(self, template):
"""
Add IPFIX template
:param template: IPFIX template
"""
templateID = template.templateID
fields = []
rec_len = 0
for field in template.templateFields:
fields.append({"name": field.informationElement, "len": field.fieldLength})
rec_len += field.fieldLength
self._templates.append({"id": templateID, "fields": fields, "rec_len": rec_len})
def decode_data_set(self, data_set):
"""
Decode IPFIX data
:param data_set: IPFIX data set
:returns: List of decoded data records.
"""
data = []
for template in self._templates:
if template["id"] == data_set.setID:
offset = 0
d = data_set[Data].data
for i in range(len(d) // template["rec_len"]):
record = {}
for field in template["fields"]:
f = d[offset : offset + field["len"]]
offset += field["len"]
record.update({field["name"]: f})
data.append(record)
break
return data
