diff options
author | 2015-06-25 10:51:15 +0300 | |
---|---|---|
committer | 2015-06-25 10:51:15 +0300 | |
commit | ef209c347bfb99ebe03a0ff2c7947a6ebbe03baf (patch) | |
tree | 51a72c0df7a8fd5d2feba6a3126d71e6cba6a34c |
first public version
-rwxr-xr-x | b | 6 | ||||
-rwxr-xr-x | images/40425209_l.jpg | bin | 0 -> 340883 bytes | |||
-rwxr-xr-x | images/Intel520.png | bin | 0 -> 34642 bytes | |||
-rwxr-xr-x | images/T-Rex_vm.png | bin | 0 -> 586111 bytes | |||
-rwxr-xr-x | images/Thumbs.db | bin | 0 -> 84992 bytes | |||
-rwxr-xr-x | images/TrexConfig.png | bin | 0 -> 57986 bytes | |||
-rwxr-xr-x | images/TrexConfig_switch.png | bin | 0 -> 75038 bytes | |||
-rwxr-xr-x | images/TrexViewer.png | bin | 0 -> 65175 bytes | |||
-rwxr-xr-x | images/cisco.png | bin | 0 -> 3330 bytes | |||
-rwxr-xr-x | images/dns_wireshark.png | bin | 0 -> 12271 bytes | |||
-rwxr-xr-x | images/icons/README | 5 | ||||
-rwxr-xr-x | images/icons/Thumbs.db | bin | 0 -> 27136 bytes | |||
-rwxr-xr-x | images/icons/callouts/1.png | bin | 0 -> 329 bytes | |||
-rwxr-xr-x | images/icons/callouts/10.png | bin | 0 -> 361 bytes | |||
-rwxr-xr-x | images/icons/callouts/11.png | bin | 0 -> 565 bytes | |||
-rwxr-xr-x | images/icons/callouts/12.png | bin | 0 -> 617 bytes | |||
-rwxr-xr-x | images/icons/callouts/13.png | bin | 0 -> 623 bytes | |||
-rwxr-xr-x | images/icons/callouts/14.png | bin | 0 -> 411 bytes | |||
-rwxr-xr-x | images/icons/callouts/15.png | bin | 0 -> 640 bytes | |||
-rwxr-xr-x | images/icons/callouts/2.png | bin | 0 -> 353 bytes | |||
-rwxr-xr-x | images/icons/callouts/3.png | bin | 0 -> 350 bytes | |||
-rwxr-xr-x | images/icons/callouts/4.png | bin | 0 -> 345 bytes | |||
-rwxr-xr-x | images/icons/callouts/5.png | bin | 0 -> 348 bytes | |||
-rwxr-xr-x | images/icons/callouts/6.png | bin | 0 -> 355 bytes | |||
-rwxr-xr-x | images/icons/callouts/7.png | bin | 0 -> 344 bytes | |||
-rwxr-xr-x | images/icons/callouts/8.png | bin | 0 -> 357 bytes | |||
-rwxr-xr-x | images/icons/callouts/9.png | bin | 0 -> 357 bytes | |||
-rwxr-xr-x | images/icons/callouts/Thumbs.db | bin | 0 -> 16896 bytes | |||
-rwxr-xr-x | images/icons/caution.png | bin | 0 -> 2734 bytes | |||
-rwxr-xr-x | images/icons/example.png | bin | 0 -> 2599 bytes | |||
-rwxr-xr-x | images/icons/home.png | bin | 0 -> 1340 bytes | |||
-rwxr-xr-x | images/icons/important.png | bin | 0 -> 2980 bytes | |||
-rwxr-xr-x | images/icons/next.png | bin | 0 -> 1302 bytes | |||
-rwxr-xr-x | images/icons/note.png | bin | 0 -> 2494 bytes | |||
-rwxr-xr-x | images/icons/prev.png | bin | 0 -> 1348 bytes | |||
-rwxr-xr-x | images/icons/tip.png | bin | 0 -> 2718 bytes | |||
-rwxr-xr-x | images/icons/up.png | bin | 0 -> 1320 bytes | |||
-rwxr-xr-x | images/icons/warning.png | bin | 0 -> 3214 bytes | |||
-rwxr-xr-x | images/ip_allocation.png | bin | 0 -> 17687 bytes | |||
-rwxr-xr-x | images/loopback_right.png | bin | 0 -> 96274 bytes | |||
-rwxr-xr-x | images/loopback_wrong.png | bin | 0 -> 97046 bytes | |||
-rwxr-xr-x | images/sfr_profile.png | bin | 0 -> 16776 bytes | |||
-rwxr-xr-x | images/small.jpg | bin | 0 -> 6943 bytes | |||
-rwxr-xr-x | images/smallnew.png | bin | 0 -> 292 bytes | |||
-rwxr-xr-x | images/trex2.png | bin | 0 -> 59226 bytes | |||
-rwxr-xr-x | images/trex_algo.png | bin | 0 -> 24137 bytes | |||
-rwxr-xr-x | images/trex_auto_script.jpg | bin | 0 -> 201947 bytes | |||
-rwxr-xr-x | images/trex_control_plane_modules.png | bin | 0 -> 455128 bytes | |||
-rwxr-xr-x | images/trex_desing.png | bin | 0 -> 17994 bytes | |||
-rwxr-xr-x | images/trex_generator_1.PNG | bin | 0 -> 24213 bytes | |||
-rwxr-xr-x | images/trex_logo.png | bin | 0 -> 56958 bytes | |||
-rwxr-xr-x | images/trex_logo_64_64.png | bin | 0 -> 27300 bytes | |||
-rwxr-xr-x | images/trex_logo_green_small.png | bin | 0 -> 55947 bytes | |||
-rwxr-xr-x | images/trex_model.png | bin | 0 -> 98328 bytes | |||
-rwxr-xr-x | images/trex_motinor_config.png | bin | 0 -> 21573 bytes | |||
-rwxr-xr-x | images/trex_motinor_view.png | bin | 0 -> 168606 bytes | |||
-rwxr-xr-x | images/trex_sfr_profile.png | bin | 0 -> 41114 bytes | |||
-rwxr-xr-x | images/trex_vm_bios_err.png | bin | 0 -> 151827 bytes | |||
-rwxr-xr-x | images/trex_vm_login.png | bin | 0 -> 49907 bytes | |||
-rwxr-xr-x | images/trex_vm_run.png | bin | 0 -> 19953 bytes | |||
-rwxr-xr-x | images/ucs200_2.png | bin | 0 -> 23528 bytes | |||
-rwxr-xr-x | images/vm_import.png | bin | 0 -> 105388 bytes | |||
-rwxr-xr-x | images/vm_selection_screen.png | bin | 0 -> 66700 bytes | |||
-rwxr-xr-x | my_chart.js | 84 | ||||
-rwxr-xr-x | release_notes.asciidoc | 25 | ||||
-rwxr-xr-x | symbols.lang | 5 | ||||
-rwxr-xr-x | trex_book-docinfo.html | 22 | ||||
-rwxr-xr-x | trex_book.asciidoc | 1232 | ||||
-rwxr-xr-x | trex_book_basic.asciidoc | 3308 | ||||
-rwxr-xr-x | trex_config.asciidoc | 299 | ||||
-rwxr-xr-x | trex_control_plane_design_phase1.asciidoc | 516 | ||||
-rwxr-xr-x | trex_control_plane_peek.asciidoc | 225 | ||||
-rwxr-xr-x | trex_preso.asciidoc | 1312 | ||||
-rwxr-xr-x | trex_vm_manual.asciidoc | 324 | ||||
-rwxr-xr-x | waf | bin | 0 -> 90909 bytes | |||
-rwxr-xr-x | waf.css | 39 | ||||
-rwxr-xr-x | wscript | 180 |
77 files changed, 7582 insertions, 0 deletions
@@ -0,0 +1,6 @@ +#! /bin/bash +python waf $@ +sts=$? +exit $sts + + diff --git a/images/40425209_l.jpg b/images/40425209_l.jpg Binary files differnew file mode 100755 index 00000000..60bb3430 --- /dev/null +++ b/images/40425209_l.jpg diff --git a/images/Intel520.png b/images/Intel520.png Binary files differnew file mode 100755 index 00000000..ad67f3ec --- /dev/null +++ b/images/Intel520.png diff --git a/images/T-Rex_vm.png b/images/T-Rex_vm.png Binary files differnew file mode 100755 index 00000000..53340b9a --- /dev/null +++ b/images/T-Rex_vm.png diff --git a/images/Thumbs.db b/images/Thumbs.db Binary files differnew file mode 100755 index 00000000..b775040b --- /dev/null +++ b/images/Thumbs.db diff --git a/images/TrexConfig.png b/images/TrexConfig.png Binary files differnew file mode 100755 index 00000000..1b956276 --- /dev/null +++ b/images/TrexConfig.png diff --git a/images/TrexConfig_switch.png b/images/TrexConfig_switch.png Binary files differnew file mode 100755 index 00000000..245a849b --- /dev/null +++ b/images/TrexConfig_switch.png diff --git a/images/TrexViewer.png b/images/TrexViewer.png Binary files differnew file mode 100755 index 00000000..ad76dcc1 --- /dev/null +++ b/images/TrexViewer.png diff --git a/images/cisco.png b/images/cisco.png Binary files differnew file mode 100755 index 00000000..1a2b826c --- /dev/null +++ b/images/cisco.png diff --git a/images/dns_wireshark.png b/images/dns_wireshark.png Binary files differnew file mode 100755 index 00000000..2d8010fc --- /dev/null +++ b/images/dns_wireshark.png diff --git a/images/icons/README b/images/icons/README new file mode 100755 index 00000000..f12b2a73 --- /dev/null +++ b/images/icons/README @@ -0,0 +1,5 @@ +Replaced the plain DocBook XSL admonition icons with Jimmac's DocBook +icons (http://jimmac.musichall.cz/ikony.php3). I dropped transparency +from the Jimmac icons to get round MS IE and FOP PNG incompatibilies. + +Stuart Rackham diff --git a/images/icons/Thumbs.db b/images/icons/Thumbs.db Binary files differnew file mode 100755 index 00000000..97d755f2 --- /dev/null +++ b/images/icons/Thumbs.db diff --git a/images/icons/callouts/1.png b/images/icons/callouts/1.png Binary files differnew file mode 100755 index 00000000..7d473430 --- /dev/null +++ b/images/icons/callouts/1.png diff --git a/images/icons/callouts/10.png b/images/icons/callouts/10.png Binary files differnew file mode 100755 index 00000000..997bbc82 --- /dev/null +++ b/images/icons/callouts/10.png diff --git a/images/icons/callouts/11.png b/images/icons/callouts/11.png Binary files differnew file mode 100755 index 00000000..ce47dac3 --- /dev/null +++ b/images/icons/callouts/11.png diff --git a/images/icons/callouts/12.png b/images/icons/callouts/12.png Binary files differnew file mode 100755 index 00000000..31daf4e2 --- /dev/null +++ b/images/icons/callouts/12.png diff --git a/images/icons/callouts/13.png b/images/icons/callouts/13.png Binary files differnew file mode 100755 index 00000000..14021a89 --- /dev/null +++ b/images/icons/callouts/13.png diff --git a/images/icons/callouts/14.png b/images/icons/callouts/14.png Binary files differnew file mode 100755 index 00000000..64014b75 --- /dev/null +++ b/images/icons/callouts/14.png diff --git a/images/icons/callouts/15.png b/images/icons/callouts/15.png Binary files differnew file mode 100755 index 00000000..0d65765f --- /dev/null +++ b/images/icons/callouts/15.png diff --git a/images/icons/callouts/2.png b/images/icons/callouts/2.png Binary files differnew file mode 100755 index 00000000..5d09341b --- /dev/null +++ b/images/icons/callouts/2.png diff --git a/images/icons/callouts/3.png b/images/icons/callouts/3.png Binary files differnew file mode 100755 index 00000000..ef7b7004 --- /dev/null +++ b/images/icons/callouts/3.png diff --git a/images/icons/callouts/4.png b/images/icons/callouts/4.png Binary files differnew file mode 100755 index 00000000..adb8364e --- /dev/null +++ b/images/icons/callouts/4.png diff --git a/images/icons/callouts/5.png b/images/icons/callouts/5.png Binary files differnew file mode 100755 index 00000000..4d7eb460 --- /dev/null +++ b/images/icons/callouts/5.png diff --git a/images/icons/callouts/6.png b/images/icons/callouts/6.png Binary files differnew file mode 100755 index 00000000..0ba694af --- /dev/null +++ b/images/icons/callouts/6.png diff --git a/images/icons/callouts/7.png b/images/icons/callouts/7.png Binary files differnew file mode 100755 index 00000000..472e96f8 --- /dev/null +++ b/images/icons/callouts/7.png diff --git a/images/icons/callouts/8.png b/images/icons/callouts/8.png Binary files differnew file mode 100755 index 00000000..5e60973c --- /dev/null +++ b/images/icons/callouts/8.png diff --git a/images/icons/callouts/9.png b/images/icons/callouts/9.png Binary files differnew file mode 100755 index 00000000..a0676d26 --- /dev/null +++ b/images/icons/callouts/9.png diff --git a/images/icons/callouts/Thumbs.db b/images/icons/callouts/Thumbs.db Binary files differnew file mode 100755 index 00000000..2312a5bb --- /dev/null +++ b/images/icons/callouts/Thumbs.db diff --git a/images/icons/caution.png b/images/icons/caution.png Binary files differnew file mode 100755 index 00000000..9a8c515a --- /dev/null +++ b/images/icons/caution.png diff --git a/images/icons/example.png b/images/icons/example.png Binary files differnew file mode 100755 index 00000000..1199e864 --- /dev/null +++ b/images/icons/example.png diff --git a/images/icons/home.png b/images/icons/home.png Binary files differnew file mode 100755 index 00000000..37a5231b --- /dev/null +++ b/images/icons/home.png diff --git a/images/icons/important.png b/images/icons/important.png Binary files differnew file mode 100755 index 00000000..be685cc4 --- /dev/null +++ b/images/icons/important.png diff --git a/images/icons/next.png b/images/icons/next.png Binary files differnew file mode 100755 index 00000000..64e126bd --- /dev/null +++ b/images/icons/next.png diff --git a/images/icons/note.png b/images/icons/note.png Binary files differnew file mode 100755 index 00000000..7c1f3e2f --- /dev/null +++ b/images/icons/note.png diff --git a/images/icons/prev.png b/images/icons/prev.png Binary files differnew file mode 100755 index 00000000..3e8f12fe --- /dev/null +++ b/images/icons/prev.png diff --git a/images/icons/tip.png b/images/icons/tip.png Binary files differnew file mode 100755 index 00000000..f087c73b --- /dev/null +++ b/images/icons/tip.png diff --git a/images/icons/up.png b/images/icons/up.png Binary files differnew file mode 100755 index 00000000..2db1ce62 --- /dev/null +++ b/images/icons/up.png diff --git a/images/icons/warning.png b/images/icons/warning.png Binary files differnew file mode 100755 index 00000000..d41edb9a --- /dev/null +++ b/images/icons/warning.png diff --git a/images/ip_allocation.png b/images/ip_allocation.png Binary files differnew file mode 100755 index 00000000..023706ef --- /dev/null +++ b/images/ip_allocation.png diff --git a/images/loopback_right.png b/images/loopback_right.png Binary files differnew file mode 100755 index 00000000..1891d25a --- /dev/null +++ b/images/loopback_right.png diff --git a/images/loopback_wrong.png b/images/loopback_wrong.png Binary files differnew file mode 100755 index 00000000..92602a86 --- /dev/null +++ b/images/loopback_wrong.png diff --git a/images/sfr_profile.png b/images/sfr_profile.png Binary files differnew file mode 100755 index 00000000..da9e5e62 --- /dev/null +++ b/images/sfr_profile.png diff --git a/images/small.jpg b/images/small.jpg Binary files differnew file mode 100755 index 00000000..1557f126 --- /dev/null +++ b/images/small.jpg diff --git a/images/smallnew.png b/images/smallnew.png Binary files differnew file mode 100755 index 00000000..411c2e17 --- /dev/null +++ b/images/smallnew.png diff --git a/images/trex2.png b/images/trex2.png Binary files differnew file mode 100755 index 00000000..2bed6ab3 --- /dev/null +++ b/images/trex2.png diff --git a/images/trex_algo.png b/images/trex_algo.png Binary files differnew file mode 100755 index 00000000..6da98473 --- /dev/null +++ b/images/trex_algo.png diff --git a/images/trex_auto_script.jpg b/images/trex_auto_script.jpg Binary files differnew file mode 100755 index 00000000..f4cdb109 --- /dev/null +++ b/images/trex_auto_script.jpg diff --git a/images/trex_control_plane_modules.png b/images/trex_control_plane_modules.png Binary files differnew file mode 100755 index 00000000..c4978e38 --- /dev/null +++ b/images/trex_control_plane_modules.png diff --git a/images/trex_desing.png b/images/trex_desing.png Binary files differnew file mode 100755 index 00000000..64b3ed1c --- /dev/null +++ b/images/trex_desing.png diff --git a/images/trex_generator_1.PNG b/images/trex_generator_1.PNG Binary files differnew file mode 100755 index 00000000..8b0d57a8 --- /dev/null +++ b/images/trex_generator_1.PNG diff --git a/images/trex_logo.png b/images/trex_logo.png Binary files differnew file mode 100755 index 00000000..3bf29278 --- /dev/null +++ b/images/trex_logo.png diff --git a/images/trex_logo_64_64.png b/images/trex_logo_64_64.png Binary files differnew file mode 100755 index 00000000..286228b6 --- /dev/null +++ b/images/trex_logo_64_64.png diff --git a/images/trex_logo_green_small.png b/images/trex_logo_green_small.png Binary files differnew file mode 100755 index 00000000..0cc7221a --- /dev/null +++ b/images/trex_logo_green_small.png diff --git a/images/trex_model.png b/images/trex_model.png Binary files differnew file mode 100755 index 00000000..1bdcfca4 --- /dev/null +++ b/images/trex_model.png diff --git a/images/trex_motinor_config.png b/images/trex_motinor_config.png Binary files differnew file mode 100755 index 00000000..70f705ce --- /dev/null +++ b/images/trex_motinor_config.png diff --git a/images/trex_motinor_view.png b/images/trex_motinor_view.png Binary files differnew file mode 100755 index 00000000..855dc027 --- /dev/null +++ b/images/trex_motinor_view.png diff --git a/images/trex_sfr_profile.png b/images/trex_sfr_profile.png Binary files differnew file mode 100755 index 00000000..f90caa42 --- /dev/null +++ b/images/trex_sfr_profile.png diff --git a/images/trex_vm_bios_err.png b/images/trex_vm_bios_err.png Binary files differnew file mode 100755 index 00000000..3ac2da5d --- /dev/null +++ b/images/trex_vm_bios_err.png diff --git a/images/trex_vm_login.png b/images/trex_vm_login.png Binary files differnew file mode 100755 index 00000000..388ee4ba --- /dev/null +++ b/images/trex_vm_login.png diff --git a/images/trex_vm_run.png b/images/trex_vm_run.png Binary files differnew file mode 100755 index 00000000..86d1cf19 --- /dev/null +++ b/images/trex_vm_run.png diff --git a/images/ucs200_2.png b/images/ucs200_2.png Binary files differnew file mode 100755 index 00000000..52cb10f4 --- /dev/null +++ b/images/ucs200_2.png diff --git a/images/vm_import.png b/images/vm_import.png Binary files differnew file mode 100755 index 00000000..e3b28647 --- /dev/null +++ b/images/vm_import.png diff --git a/images/vm_selection_screen.png b/images/vm_selection_screen.png Binary files differnew file mode 100755 index 00000000..a2c8e19b --- /dev/null +++ b/images/vm_selection_screen.png diff --git a/my_chart.js b/my_chart.js new file mode 100755 index 00000000..41851e0f --- /dev/null +++ b/my_chart.js @@ -0,0 +1,84 @@ +function chart(id,data,data_names,xlabel,ylabel){ + +var margin = {_myt: 20, _right: 20, _bottom: 30, _left: 40}; + +var width = 960 - margin._left - margin._right; +var height = 500 - margin._myt - margin._bottom; + +var x = d3.scale.linear() + .range([0, width]); + +var y = d3.scale.linear() + .range([height, 0]); + +var color = d3.scale.category10(); + +var xAxis = d3.svg.axis() + .scale(x) + .orient("bottom"); + +var yAxis = d3.svg.axis() + .scale(y) + .orient("left"); + +var svg = d3.select(id).append("svg") + .attr("width", width + margin._left + margin._right) + .attr("height", height + margin._myt + margin._bottom) + .append("g") + .attr("transform", "translate(" + margin._left + "," + margin._myt + ")"); + +x.domain(d3.extent(data, function(d) { return d[0] })).nice(); +y.domain(d3.extent(data, function(d) { return d[1] })).nice(); + +svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(xAxis) + .append("text") + .attr("class", "label") + .attr("x", width) + .attr("y", -6) + .style("text-anchor", "end") + .text(xlabel); + +svg.append("g") + .attr("class", "y axis") + .call(yAxis) + .append("text") + .attr("class", "label") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text(ylabel) + +svg.selectAll(".dot") + .data(data) + .enter().append("circle") + .attr("class", "dot") + .attr("r", 3.5) + .attr("cx", function(d) { return x(d[0]); }) + .attr("cy", function(d) { return y(d[1]); }) + .style("fill", function(d) { return color(d[2]); }); + +var legend = svg.selectAll(".legend") + .data(color.domain()) + .enter().append("g") + .attr("class", "legend") + .attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; }); + +legend.append("rect") + .attr("x", width - 18) + .attr("width", 18) + .attr("height", 18) + .style("fill", color); + +legend.append("text") + .attr("x", width - 24) + .attr("y", 9) + .attr("dy", ".35em") + .style("text-anchor", "end") + .text(function(d) { return data_names[d]; }); + +} + diff --git a/release_notes.asciidoc b/release_notes.asciidoc new file mode 100755 index 00000000..4300ba06 --- /dev/null +++ b/release_notes.asciidoc @@ -0,0 +1,25 @@ +:author: hhaim +:email: <hhaim@cisco.com> + + +ifndef::backend-docbook[] +++++++++++++++ +<div id="header-pic" style="padding:50px;margin-top:0px;position:absolute;left:0px;width:100%;"> + <img src="images/trex_logo.png" alt="Smiley face" height="80" width="270"/> +</div> +<br/><br/><br/><br/><br/><br/><br/><br/><br/><br/> +++++++++++++++ + +== TRex release notes == +endif::backend-docbook[] + +ifdef::backend-docbook[] + +== TRex release notes == +:numbered: + +endif::backend-docbook[] + +== Release 1.72 == + + diff --git a/symbols.lang b/symbols.lang new file mode 100755 index 00000000..38ac4e1c --- /dev/null +++ b/symbols.lang @@ -0,0 +1,5 @@ +co_symbol = "<1>","<2>","<3>","<4>","<5>","<6>","<7>","<8>", + "<9>","<10>","<11>","<12>","<13>","<14>","<15>" +symbol = "~","!","%","^","*","(",")","-","+","=","[", + "]","\\",":",";",",",".","/","?","&","<",">","\|" + diff --git a/trex_book-docinfo.html b/trex_book-docinfo.html new file mode 100755 index 00000000..a444f506 --- /dev/null +++ b/trex_book-docinfo.html @@ -0,0 +1,22 @@ + +<script src="http://d3js.org/d3.v3.min.js" charset="utf-8"></script> + +<script src="my_chart.js"></script> + +<style> +.axis path, +.axis line { + fill: none; + stroke: #000; + shape-rendering: crispEdges; +} + +.dot { + stroke: #000; +} +</style> + + + + + diff --git a/trex_book.asciidoc b/trex_book.asciidoc new file mode 100755 index 00000000..5e381356 --- /dev/null +++ b/trex_book.asciidoc @@ -0,0 +1,1232 @@ +TRex +==== +:author: hhaim +:email: <hhaim@cisco.com> +:revnumber: 1.70-0.0 +:quotes.++: +:numbered: +:web_server_url: http://trex-tgn.cisco.com/trex + + + + +== Introduction + +=== A word on traffic generators + +Traditionally, routers have been tested using commercial traffic generators, while performance +typically has been measured using packets per second (PPS) metrics. As router functionality and +services have become more complex, stateful traffic generators have become necessary to +provide more realistic application traffic scenarios. +The advantages of realistic traffic generators are: + +* Providing more accurate performance numbers +* Finding real bottlenecks + +==== Current Challenges: + +* *Cost* : Commercial State-full traffic generators are expensive +* *Scale* : Bandwidth does not scale up well with features complexity +* *Standardization* : Lack of standardization of traffic patterns and methodologies +* *Flexibility* : Commercial tools do not allow agility when flexibility and changes are needed + +==== Implications + +* High capital expenditure (capex) spent by different teams +* Testing in low scale and extrapolation became a common practice, it is not accurate, and hides real life bottlenecks and quality issues +* Different feature / platform teams benchmark and results methodology +* Delays in development and testing due to testing tools features dependency +* Resource and effort investment in developing different ad hoc tools and test methodologies + +=== Overview of TRex + +TRex addresses these problems through an innovative and extendable software implementation and by leveraging standard and open SW and x86/UCS HW. + +* Generates and analyzes L4-7 traffic and able to provide in one tool capabilities provided by commercial L7 tools. +* Stateful traffic generator based on pre-processing and smart replay of real traffic templates. +* Generates and *amplifies* both client and server side traffic. +* Customized functionality can be added. +* Scale to 200Gb/sec for one UCS ( using Intel 40Gb/sec NICS) +* Low cost +* Virtual interfaces support, enable TRex to be used in a fully virtual environment without physical NICs and the following example use cases: +** Amazon AWS +** Cisco LaaS +** TRex on your laptop +** Self-contained packaging that can be easily installed and deployed + + +.TRex Hardware +[options="header",cols="1^,1^"] +|================= +|Cisco UCS Platform | Intel NIC +| image:images/ucs200_2.png[title="generator"] | image:images/Intel520.png[title="generator"] +|================= + +=== Purpose of this guide + +This guide explains the use of TRex internals and the use of TRex in conjunction with Cisco ASR1000 Series routers. The examples illustrate novel traffic generation techniques made possible by TRex. + +== Download and installation + +=== Hardware recommendation + +TRex operates in a Linux application environment, interacting with Linux kernel modules. +TRex curretly works on x86 architecture and can operates well on Cisco UCS hardware. The following platforms have been tested and are recommended for operating TRex. + +[NOTE] +===================================== + A high-end UCS platform is not required for operating TRex in its current version, but may be required for future versions. +===================================== + + +.Preferred UCS +[options="header",cols="1,3"] +|================= +| UCS Type | Comments +| UCS C220 M3/M4 | Supports up to 40Gb/sec with 540-D2 and with newer Intel NIC 80Gb/sec with 1RU, recommended +| UCS C200| Early UCS model +| UCS C210 M2 | Supports up to 40Gb/sec PCIe3.0 +| UCS C240 M3 | Supports up to 200Gb/sec using Intel XL710 NICS +| UCS C260M2 | Supports up to 30Gb/sec due to V2 PCIe. +|================= + +.Internal Components +[options="header",cols="1,2",width="60%"] +|================= +| Components | Details +| CPU | 2x CPU E5-2620 +| CPU Configuration | 2-Socket CPU configurations (can also work with one CPU) +| Memory | 2x4 banks for each CPU. Total of 8 BANKS ==> 32GB +| NO RAID | NO RAID +|================= + +.Intel NICS supported +[options="header",cols="1,1,2",width="50%"] +|================= +| Bandwidth | Chipset | Example +| 1Gb/sec | Intel I350 | Intel 4x1GE 350-T4 NIC +| 10Gb/sec | Intel 82599 | Intel x520-D2 Cisco Order tool 2X Intel N2XX-AIPCI01, Intel X520 Dual Port 10Gb SFP+ Adapter +| 40Gb/sec | Intel XL710 | QSFP+ +| VMXNET | VMware paravirtualize | connect using vmWare vSwitch +| E1000 | paravirtualize | vmWare/KVM/VirtualBox +|================= + +IMPORTANT: Intel SFP+ 10Gb/Sec is the only one supported by default on the standard Linux driver. TRex also supports Cisco 10Gb/sec SFP+. + +.Sample order for UCSC-C220-M3S with 4x10Gb ports +[options="header",cols="2,1^",width="50%"] +|================= +| Component | Amount +| UCSC-C220-M3S | 1 +| UCS-CPU-E5-2650 | 2 +| UCS-MR-1X041RY-A | 8 +| A03-D500GC3 | 1 +| N2XX-AIPCI01 | 2 +| UCSC-PSU-650W | 1 +| SFS-250V-10A-IS | 1 +| UCSC-CMA1 | 1 +| UCSC-HS-C220M3 | 2 +| N20-BBLKD | 7 +| UCSC-PSU-BLKP | 1 +| UCSC-RAIL1 | 1 +|======================== + +NOTE: You should buy seperatly the 10Gb/sec SFP+, Cisco would be fine with TRex ( but not for plain Linux driver ). + +=== Install OS + +==== Supported versions + +Fedora 18, Ubuntu 14.04.1 LTS and Fedora 20 are the Linux OS supported. +More OS could be supported by compiling the drivers. + +==== Download ISO file + +Download the ISO from Fedora web site from link:http://archive.fedoraproject.org/pub/fedora/linux/releases/18/Fedora/x86_64/iso/[here]. + +Verify the checksum with the following command: + +[source,bash] +---- +$sha256sum Fedora-18-x86_64-DVD.iso +91c5f0aca391acf76a047e284144f90d66d3d5f5dcd26b01f368a43236832c03 #<1> +---- +<1> Should be equal to this number. + +==== Install Linux + +Ask your lab admin to install the Linux using CIMC, assign an IP, and set the DNS. Request the sudo or super user password to enable you to ping and SSH. + +IMPORTANT: To use TRex, you should have sudo on this machine or root password. +WARNING: Upgrading the linux Kernel using `yum upgrade` require to build the TRex drivers. + +==== Verify Intel NIC installation + +The following is an example of 4x10Gb/sec TRex with I350 management port and four x520-D2 (82599 chipset): + +[source,bash] +---- +$[root@trex]lspci | grep Network +01:00.0 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01) #<1> +01:00.1 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01) #<2> +03:00.0 Ethernet controller: Intel Corporation 82599EB 10-Gigabit SFI/SFP+ Network Connection (rev 01) #<3> +03:00.1 Ethernet controller: Intel Corporation 82599EB 10-Gigabit SFI/SFP+ Network Connection (rev 01) +82:00.0 Ethernet controller: Intel Corporation 82599EB 10-Gigabit SFI/SFP+ Network Connection (rev 01) +82:00.1 Ethernet controller: Intel Corporation 82599EB 10-Gigabit SFI/SFP+ Network Connection (rev 01) +---- +<1> Management port +<2> CIMC port +<3> 10Gb/sec traffic ports ( Intel 82599EB) + +=== Obtaining the TRex package + +Connect by ssh to the TRex machine and do the following: + +assuming *$WEB_URL* is *{web_server_url}* + +[source,bash] +---- +$mkdir trex +$cd trex +$wget --no-cache $WEB_URL/release/latest +$tar -xzvf latest +---- + + +to take the bleeding edge version +[source,bash] +---- +$wget --no-cache $WEB_URL/release/be_latest +---- + +To obtain a specific version, do the following: +[source,bash] +---- +$wget --no-cache $WEB_URL/release/vX.XX.tar.gz #<1> +---- + +=== Running TRex for the first time in loopback + +If you have 10Gb/sec TRex (based on Intel 520-D2 NICs) you can verify that it works correctly by loopback the ports. +You can install Intel SFP+ or Cisco SFP+, but you cannot connect ports that are on the same NIC to each other (it might not sync). +If you have only one NIC of 10gb/sec you cannot perform this test beacause the ports will not have valid link. +Another option for loopback is to use Cisco twinax copper cable see link:http://www.fiberopticshare.com/tag/cisco-10g-twinax[here] + +//TBD: perhaps rephase, using a "Prerequisites" or "Required" heading. The requirement here would be: Two (2) 10gb/sec NICs +//[hh] it is not accurate beacuse with 1Gb/sec you can have this test + +.Correct loopback +image:images/loopback_right.png[title="rigt"] + +.Wrong loopback +image:images/loopback_wrong.png[title="rigt"] + +In case you have 1Gb/Sec Intel NIC (I350) you can do anything you like from the loopback perspective *but* you must filter the management port before see xref:trex_config[here]. + +==== Identify the ports + +[source,bash] +---- + $>sudo ./dpdk_setup_ports.py --s + + Network devices using DPDK-compatible driver + ============================================ + + Network devices using kernel driver + =================================== + 0000:02:00.0 '82545EM Gigabit Ethernet Controller (Copper)' if=eth2 drv=e1000 unused=igb_uio *Active* + 0000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv= unused=ixgb #<1> + 0000:03:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv= unused=ixgb #<2> + 0000:13:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv= unused=ixgb #<3> + 0000:13:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv= unused=ixgb #<4> + + + Other network devices + ===================== + <none> +---- +<1> TRex interface #1 before unbinding +<2> TRex interface #2 before unbinding +<3> TRex interface #3 before unbinding +<4> TRex interface #4 before unbinding + +Now choose the port you want to use and follow the next section by creating a configuration file. + +==== Create minimum configuration file + +Create a configuration file in `/etc/trex_cfg.yaml`. + +You could copy a basic configuration file from cfg folder by running this command. + +[source,bash] +---- +$cp cfg/vm1.yaml /etc/trex_cfg.yaml +---- + +Now edit the configuration file with the right values from the previous section + +[source,bash] +---- +<none> +- port_limit : 4 #<1> + version : 2 #<2> + interfaces : ["03:00.0","03:00.1","13:00.1","13:00.0"] #<3> +---- +<1> the number of ports +<2> must add version 2 to the configuration file +<3> The list of interface from `#>sudo ./dpdk_setup_ports.py -s`, in this example it was taken + +When working with VM, you must set the destination mac of one port as the source or the other for loopback the port in the vSwitch +and you should take the right value from the hypervisor (in case of a physical NIC you can set the mac-address with virtual you can't and you should take it from the hypervisor) +and example + +[source,python] +---- + - port_limit : 2 + version : 2 + interfaces : ["03:00.0","03:00.1"] + port_info : # set eh mac addr + - dest_mac : [0x1,0x0,0x0,0x1,0x0,0x00] # port 0 + src_mac : [0x2,0x0,0x0,0x2,0x0,0x00] <1> + - dest_mac : [0x2,0x0,0x0,0x2,0x0,0x00] # port 1 <1> + src_mac : [0x1,0x0,0x0,0x1,0x0,0x00] +---- +<1> source mac is like destination mac (this should be set or taken from vmware). the mac was taken from hypervisor + +==== Running TRex + +Run this for 4x10Gb/sec TRex: +[source,bash] +---- +$sudo ./t-rex-64 -f cap2/dns.yaml -c 4 -m 1 -d 100 -l 1000 +---- + +NOTE: For 10Gb/sec TRex with 2,6, or 8 ports, add --limit-ports [number of ports] *or* follow xref:trex_config[this] to configure the TRex. +//TBD: recommend bold for the 2 commands. + +If successful, the output will be similar to the following: + +[source,python] +---- +$ sudo ./t-rex-64 -f cap2/dns.yaml -d 100 -l 1000 +Starting T-Rex 1.50 please wait ... +zmq publisher at: tcp://*:4500 + number of ports founded : 4 + port : 0 + ------------ + link : link : Link Up - speed 10000 Mbps - full-duplex <1> + promiscuous : 0 + port : 1 + ------------ + link : link : Link Up - speed 10000 Mbps - full-duplex + promiscuous : 0 + port : 2 + ------------ + link : link : Link Up - speed 10000 Mbps - full-duplex + promiscuous : 0 + port : 3 + ------------ + link : link : Link Up - speed 10000 Mbps - full-duplex + promiscuous : 0 + + + -Per port stats table + ports | 0 | 1 | 2 | 3 + ------------------------------------------------------------------------------------- + opackets | 1003 | 1003 | 1002 | 1002 + obytes | 66213 | 66229 | 66132 | 66132 + ipackets | 1003 | 1003 | 1002 | 1002 + ibytes | 66225 | 66209 | 66132 | 66132 + ierrors | 0 | 0 | 0 | 0 + oerrors | 0 | 0 | 0 | 0 + Tx Bw | 217.09 Kbps | 217.14 Kbps | 216.83 Kbps | 216.83 Kbps + + -Global stats enabled + Cpu Utilization : 0.0 % <12> 29.7 Gb/core <13> + Platform_factor : 1.0 + Total-Tx : 867.89 Kbps <2> + Total-Rx : 867.86 Kbps <3> + Total-PPS : 1.64 Kpps + Total-CPS : 0.50 cps + + Expected-PPS : 2.00 pps <9> + Expected-CPS : 1.00 cps <10> + Expected-BPS : 1.36 Kbps <11> + + Active-flows : 0 <6> Clients : 510 Socket-util : 0.0000 % + Open-flows : 1 <7> Servers : 254 Socket : 1 Socket/Clients : 0.0 + drop-rate : 0.00 bps <8> + current time : 5.3 sec + test duration : 94.7 sec + + -Latency stats enabled + Cpu Utilization : 0.2 % <14> + if| tx_ok , rx_ok , rx ,error, average , max , Jitter , max window + | , , check, , latency(usec),latency (usec) ,(usec) , + -------------------------------------------------------------------------------------------------- + 0 | 1002, 1002, 0, 0, 51 , 69, 0 | 0 69 67 <4> + 1 | 1002, 1002, 0, 0, 53 , 196, 0 | 0 196 53 <5> + 2 | 1002, 1002, 0, 0, 54 , 71, 0 | 0 71 69 + 3 | 1002, 1002, 0, 0, 53 , 193, 0 | 0 193 52 +---- +<1> Link must be up for TRex to work. +<2> Total Rx must be the same as Tx +<3> Total Rx must be the same as Tx +<4> Tx_ok == Rx_ok +<5> Tx_ok == Rx_ok +<6> Number of TRex active "flows". Could be diffrent than the Router flows due to aging issues. Usualy TRex number of active flows is much lower that router. +<7> Number of TRex flows from startup. +<8> Drop rate. +<9> Expected Packet Per Second (without the latency packets). +<10> Expected Connection Per Second (without the latency packets). +<11> Expected Bit Per Second (without the latency packets). +<12> Average CPU utilization of transmitters threads. For best results it should be lower than 80%. +<13> Gb/sec generated per core of DP. Higer is better. +<14> Rx and latency thread CPU utilization. + +WARNING: if you don't see rx packets, revisit your mac-address configuration. + +==== Running TRex for the first time with router + +You can follow this presentation link:trex_config_guide.html[first time TRex configuration] +//TBD: Note that the link does not work correctly in PDF rendition +or continue reading. +TRex set source-mac of all port to `00:00:00:01:00:00` and expected to get to this MAC-address `00:00:00:01:00:00` without a config file. +so you just need to configure router with the right mac-address. + +NOTE: Virtual routers on ESXi (for example, Cisco CSR1000v) must have a distinct MAC address for each port. Specify the address in the configuration file. see more xref:trex_config[here]. Another example is where the TRex is connected to a switch. In that case each of TRex port should have a distinc MAC address. + +include::trex_book_basic.asciidoc[] + +== Advanced features + + +=== VLAN Trunk support anchor:trex_valn[] + +The VLAN Trunk TRex feature attempts to solve the router port bandwidth limitation when the traffic profile is asymmetric. Example: SFR profile is asymmetric and was the first usecase. +This feature converts asymmetric traffic to symmetric, from the port perspective, using router sub-interfaces. +This feature requires TRex to send the traffic on two VLANs. The following describes how this works. + +.YAML format +[source,python] +---- + vlan : { enable : 1 , vlan0 : 100 , vlan1 : 200 } +---- + + +.Example +[source,python] +---- +- duration : 0.1 + vlan : { enable : 1 , vlan0 : 100 , vlan1 : 200 } <1> +---- +<1> enable VLAN feature , valn0==100 , valn1==200 + +*Problem definition:*:: + +Assuming a TRex with two ports and an SFR traffic profile. + +.Without VLAN/sub interfaces +[source,python] +---- +0 ( client) -> [ ] - 1 ( server) +---- +Without VLAN support it is not symmetric. From port 0 (client side), it sends 10%, from and port 1 (server) sends 90%. Port 1 become the bottlneck (10Gb/s limit) before port 0 + +.With VLAN/sub interfaces +[source,python] +---- +port 0 ( client VLAN0) <-> | | <-> port 1 ( server-VLAN0) +port 0 ( server VLAN1) <-> | | <-> port 1 ( client-VLAN1) +---- + +In this case both ports will have the same amount of traffic. + +*Router configuation:*:: +[source,python] +---- + ! + interface TenGigabitEthernet1/0/0 <1> + mac-address 0000.0001.0000 + mtu 4000 + no ip address + load-interval 30 + ! + i + interface TenGigabitEthernet1/0/0.100 + encapsulation dot1Q 100 <2> + ip address 11.77.11.1 255.255.255.0 + ip nbar protocol-discovery + ip policy route-map vlan_100_p1_to_p2 <3> + ! + interface TenGigabitEthernet1/0/0.200 + encapsulation dot1Q 200 <4> + ip address 11.88.11.1 255.255.255.0 + ip nbar protocol-discovery + ip policy route-map vlan_200_p1_to_p2 <5> + ! + interface TenGigabitEthernet1/1/0 + mac-address 0000.0001.0000 + mtu 4000 + no ip address + load-interval 30 + ! + interface TenGigabitEthernet1/1/0.100 + encapsulation dot1Q 100 + ip address 22.77.11.1 255.255.255.0 + ip nbar protocol-discovery + ip policy route-map vlan_100_p2_to_p1 + ! + interface TenGigabitEthernet1/1/0.200 + encapsulation dot1Q 200 + ip address 22.88.11.1 255.255.255.0 + ip nbar protocol-discovery + ip policy route-map vlan_200_p2_to_p1 + ! + + arp 11.77.11.12 0000.0001.0000 ARPA <6> + arp 22.77.11.12 0000.0001.0000 ARPA + + route-map vlan_100_p1_to_p2 permit 10 <7> + set ip next-hop 22.77.11.12 + ! + route-map vlan_100_p2_to_p1 permit 10 + set ip next-hop 11.77.11.12 + ! + + route-map vlan_200_p1_to_p2 permit 10 + set ip next-hop 22.88.11.12 + ! + route-map vlan_200_p2_to_p1 permit 10 + set ip next-hop 11.88.11.12 + ! +---- +<1> Disable the IP on the main port it is important +<2> Enable VLAN1 +<3> PBR configuration +<4> Enable VLAN2 +<5> PBR configuration +<6> TRex MAC-address destination port +<7> PBR configuration rules + +=== Static source MAC-address setting + +With this feature, TRex replaces the source MAC address with the client IP address. +Note: This feature was requested by the Cisco ISG group. + + +*YAML:*:: +[source,python] +---- + mac_override_by_ip : true +---- + +.Example +[source,python] +---- +- duration : 0.1 + .. + mac_override_by_ip : true <1> +---- +<1> In this case, the client side MAC address will be look like this: +SRC_MAC = IPV4(IP) + 00:00 + +=== IPv6 support ( `--ipv6`); + +Support for IPv6 includes: + +1. Support for pcap files containing IPv6 packets +2. Ability to generate IPv6 traffic from pcap files containing IPv4 packets +The following switch enables this feature: `--ipv6` +Two new keywords (src_ipv6, dst_ipv6) have been added to the YAML +file to specify the most significant 96-bits of the IPv6 address - for example: + +[source,python] +---- + src_ipv6 : [0xFE80,0x0232,0x1002,0x0051,0x0000,0x0000] + dst_ipv6 : [0x2001,0x0DB8,0x0003,0x0004,0x0000,0x0000] +---- + +The IPv6 address is formed by placing what would typically be the IPv4 +address into the least significant 32-bits and copying the value provided +in the src_ipv6/dst_ipv6 keywords into the most signficant 96-bits. +If src_ipv6 and dst_ipv6 are not specified in the YAML file, the default +is to form IPv4-compatible addresses (where the most signifcant 96-bits +are zero). + +There is a support for all plugins (control flows that needed to be change). + +*An example:*:: +[source,bash] +---- +$sudo ./t-rex-64 -f cap2l/sfr_delay_10_1g.yaml -c 4 -p -l 100 -d 100000 -m 30 --ipv6 +---- + +*Limitations:*:: + +* TRex cannot generate both IPv4 and IPv6 traffic. The --ipv6 switch must be specified even when using a pcap file containing only IPv6 packets + + +*Router configuration:*:: + +[source,python] +---- +interface TenGigabitEthernet1/0/0 + mac-address 0000.0001.0000 + mtu 4000 + ip address 11.11.11.11 255.255.255.0 + ip policy route-map p1_to_p2 + load-interval 30 + ipv6 enable ==> IPv6 + ipv6 address 2001:DB8:1111:2222::1/64 <1> + ipv6 policy route-map ipv6_p1_to_p2 <2> +! + + +ipv6 unicast-routing <3> + +ipv6 neighbor 3001::2 TenGigabitEthernet0/1/0 0000.0002.0002 <4> +ipv6 neighbor 2001::2 TenGigabitEthernet0/0/0 0000.0003.0002 + +route-map ipv6_p1_to_p2 permit 10 <5> + set ipv6 next-hop 2001::2 +! +route-map ipv6_p2_to_p1 permit 10 + set ipv6 next-hop 3001::2 +! + + +asr1k(config)#ipv6 route 4000::/64 2001::2 +asr1k(config)#ipv6 route 5000::/64 3001::2 +---- +<1> enable ipv6 +<2> add pbr +<3> enable ipv6 routing +<4> mac-addr setting should be like T-Rex +<5> PBR configuraion + + +=== Source MAC-address mapping using a file + +Extending the source MAC-address replacment capability. +It is possible to have a mapping betwean IPv4->MAC using the new `--mac` CLI switch +file format is YAML. + +*An example:*:: +[source,bash] +---- +$sudo ./t-rex-64 -f cap2/sfr_delay_10_1g.yaml -c 4 -l 100 -d 100000 -m 30 --mac cap2/test_example.yaml +---- + +*MAC file structure:*:: + +[source,python] +---- +- items : + - ip : "16.0.0.1" + mac : [0x16,0x1,0x4,0x5,0x6,0x7] + - ip : "16.0.0.2" + mac : [0x16,0x2,0x0,0x1,0x0,0x0] +---- + +*Limitations:*:: + +. It is assumed that most of the clients has MAC-addrees. at least 90% of the IP should have a MAC-addrees mapping. + +=== Destination mac address spreadings anchor:mac_spread[] + +Using this option, one can send traffic to a few destination devices. In normal mode all the packets are sent to the port destination mac-address. +to enable this option add this CLI `--mac-spread` to the command line + +example: + +[source,bash] +---- +$sudo ./t-rex-64 -f cap2/http_simple.yaml -d 1000 -m 1000 -c 4 -l 100 --mac-spread 2 +---- +in this case TRex will send to port destination mac and port destination mac +1 +using a switch you could connect TRex to a few DUT. +All the DUTs should return the traffic only to right port source address + +[source,bash] +---- + switch A switch A + | | + | D0+0 -> DUT0 <- D1+0 | +TRex(0) -| |-TRex(1) + | | + | D0+1 -> DUT1 <- D1+1 | + | + +---- + + +=== NAT support + +TRex can learn dynamic NAT/PAT translation. To enable this feature add `--learn` to the command line. +//TBD: maybe... add the '--learn' option on the command line. +This is done by adding an IPv4 option header with TRex info (8 bytes long 0x10 id) to the first packet of the flow. + +*Example:*:: + +*simple HTTP traffic* + +[source,bash] +---- +$sudo ./t-rex-64 -f cap2/http_simple.yaml -c 4 -l 1000 -d 100000 -m 30 --learn +---- + +*SFR traffic without bundeling/ALG support* + +[source,bash] +---- +$sudo ./t-rex-64 -f avl/sfr_delay_10_1g_no_bundeling.yaml -c 4 -l 1000 -d 100000 -m 10 --learn +---- + +*New terminal counters:*:: + +[source,python] +---- +-Global stats enabled + Cpu Utilization : 0.6 % 33.4 Gb/core + Platform_factor : 1.0 + Total-Tx : 773.76 Mbps Nat_time_out : 0 <1> + Total-Rx : 770.47 Mbps Nat_no_fid : 0 <2> + Total-PPS : 106.73 Kpps Total_nat_active: 9 <3> + Total-CPS : 2.78 Kcps Total_nat_open : 232129 <4> +---- +<1> The number of translations with timeout should be zero. Usually this occurs when the router drops the flow due to NAT. +<2> Translation not found. This can occur when there is large latency in the router input/output queue. +<3> Active number of TRex traslation flows, should be low in the case of low RTT. +<4> A total of TRex translation. May be different from the total number of flows in case template is uni-directional (no need a translation). + + +*Configuration for Cisco ASR1000 Series:*:: + +The feature was tested with the following configuration and sfr_delay_10_1g_no_bundeling. yaml traffic profile. +Clients address range is 16.0.0.1-16.0.0.255 + +[source,python] +---- +interface TenGigabitEthernet1/0/0 <1> + mac-address 0000.0001.0000 + mtu 4000 + ip address 11.11.11.11 255.255.255.0 + ip policy route-map p1_to_p2 + ip nat inside <2> + load-interval 30 +! + +interface TenGigabitEthernet1/1/0 + mac-address 0000.0001.0000 + mtu 4000 + ip address 11.11.11.11 255.255.255.0 + ip policy route-map p1_to_p2 + ip nat outside <3> + load-interval 30 + +ip nat pool my 200.0.0.0 200.0.0.255 netmask 255.255.255.0 <4> + +ip nat inside source list 7 pool my overload +access-list 7 permit 16.0.0.0 0.0.0.255 <5> + +ip nat inside source list 8 pool my overload <6> +access-list 8 permit 17.0.0.0 0.0.0.255 +---- +<1> Should be connected to TRex Client port (router inside port) +<2> NAT inside +<3> NAT outside +<4> Pool of outside address with overload +<5> Should match TRex YAML client range +<6> In case of dual port TRex. + + +*Limitations:*:: + +. The IPv6-IPv6 NAT feature does not exist on routers, so this feature can work on IPv4 only. +. Does not support NAT64. +. Bundeling/plugin support is not fully supported. This means that sfr_delay_10.yaml can't work.Use sfr_delay_10_no_bundeling.yaml instead. + +[NOTE] +===================================================================== +* `--learn-verify` is a debug TRex mechanism for testing the TRex learn mechanism. +* If the router is configured without NAT, it will verify that the inside_ip==outside_ip and inside_port==outside_port. +===================================================================== + +=== Flow order/latency verification ( `--rx-check` ) + +In normal mode (without this feature enabled), received traffic is not checked by software. It only counted by hardware (Intel NIC) for drop packets verification at test end of the test. The only exception is the Latency/Jitter packets. +This is one of the reasons that with TRex, you *cannot* check features that terminate traffic (for example TCP Proxy) +To enable this feature you should add to the command line options `--rx-check [sample]` where sample is the sample rate. +1/sample flows will be loaded to the software for verification. For 40Gb/Sec traffic you can use a sample of 1/128. Watch for Rx CPU% utilization. + +INFO : This feature changes the TTL of the sample flows to 255 and expects 254 or 255 (one routing hop). If you have more than one hop in your setup, use `--hops` to change it to higher value. more than one hop could happned when there are number of routers betwean TRex client side to TRex server side. + +With this feature enabled: + +* You can verify that packets get out of DUT in order (from each flow perspective) +* There are no dropped packets. There is no need to wait for the end of the test. Without this feature enabled you must wait for the end of the test to be aware of dropped packets because there is always a difference between TX and Rx due RTT. To be sure there is a need to stop the traffic and wait for the Rx traffic and this happens only at the end of the test. + + +.Full example +[source,bash] +---- +$sudo ./t-rex-64 -f avl/sfr_delay_10_1g.yaml -c 4 -p -l 100 -d 100000 -m 30 --rx-check 128 +---- + +[source,python] +---- +Cpu Utilization : 0.1 % <1> + if| tx_ok , rx_ok , rx ,error, average , max , Jitter<1> , max window + | , , check, , latency(usec),latency (usec) ,(usec) , + -------------------------------------------------------------------------------- + 0 | 1002, 1002, 2501, 0, 61 , 70, 3 | 60 + 1 | 1002, 1002, 2012, 0, 56 , 63, 2 | 50 + 2 | 1002, 1002, 2322, 0, 66 , 74, 5 | 68 + 3 | 1002, 1002, 1727, 0, 58 , 68, 2 | 52 + + Rx Check stats enabled <2> + ------------------------------------------------------------------------------------------- + rx check: avg/max/jitter latency, 94 , 744, 49<1> | 252 287 309 <3> + + active flows: 10, fif: 308, drop: 0, errors: 0 <4> + ------------------------------------------------------------------------------------------- +---- +<1> CPU% of the Rx thread. If it is too high *increase* the sample rate. +<2> Rx Check section. For more detailed info, press 'r' during the test or at the end of the test. +<3> Average latency, max latency, jitter on the template flows in microseconds. This is usually *higher* than the latency check packet because the feature works more on this packet. +<4> Drop counters and errors counter should be zero. If not, press 'r' to see the full report or view the report at the end of the test. + +.Full report by pressing 'r' +[source,python] +---- + m_total_rx : 2 + m_lookup : 2 + m_found : 1 + m_fif : 1 + m_add : 1 + m_remove : 1 + m_active : 0 + <1> + 0 0 0 0 1041 0 0 0 0 0 0 0 0 min_delta : 10 usec + cnt : 2 + high_cnt : 2 + max_d_time : 1041 usec + sliding_average : 1 usec + precent : 100.0 % + histogram + ----------- + h[1000] : 2 + tempate_id_ 0 , errors: 0, jitter: 61 <2> + tempate_id_ 1 , errors: 0, jitter: 0 + tempate_id_ 2 , errors: 0, jitter: 0 + tempate_id_ 3 , errors: 0, jitter: 0 + tempate_id_ 4 , errors: 0, jitter: 0 + tempate_id_ 5 , errors: 0, jitter: 0 + tempate_id_ 6 , errors: 0, jitter: 0 + tempate_id_ 7 , errors: 0, jitter: 0 + tempate_id_ 8 , errors: 0, jitter: 0 + tempate_id_ 9 , errors: 0, jitter: 0 + tempate_id_10 , errors: 0, jitter: 0 + tempate_id_11 , errors: 0, jitter: 0 + tempate_id_12 , errors: 0, jitter: 0 + tempate_id_13 , errors: 0, jitter: 0 + tempate_id_14 , errors: 0, jitter: 0 + tempate_id_15 , errors: 0, jitter: 0 + ager : + m_st_alloc : 1 + m_st_free : 0 + m_st_start : 2 + m_st_stop : 1 + m_st_handle : 0 +---- +<1> Any errors shown here +<2> Error per template info + +*Limitation:*:: + +** This feature must be enabled with a latency check (-l). +** To receive the packets TRex does the following: +*** Changes the TTL to 0xff and expects 0xFF (loopback) or oxFE (route). ( use --hop to tune this number) +*** Adds 24 bytes of metadata as ipv4/ipv6 option header + +== Reference + +=== Traffic YAML + +==== Global Traffic YAML section + +[source,python] +---- +- duration : 10.0 <1> + generator : <2> + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.255" + servers_start : "48.0.0.1" + servers_end : "48.0.0.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 1 + udp_aging : 1 + mac : [0x00,0x00,0x00,0x01,0x00,0x00] <3> + vlan : { enable : 1 , vlan0 : 100 , vlan1 : 200 } <7> + mac_override_by_ip : true <8> + cap_ipg : true <4> + cap_ipg_min : 30 <5> + cap_override_ipg : 200 <6> +---- +<1> Duration of the test (seconds). Can override using the `-d` option. +<2> See the generator section. +<3> Default source/destination MAC address. The configuration file can override the defaults. +<4> TRUE indicates that the IPG is taken from pcap file. +<5> The following two options can set the min ipg in microseconds: ( if (pkt_ipg<cap_ipg_min) { pkt_ipg=cap_override_ipg) } +<6> Value to override (microseconds). +<7> Enable valn feature. See xref:trex_valn[here] for info. +<8> Enable MAC address replacement by Client IP. + +==== Per template section + +[source,python] +---- + - name: cap2/dns.pcap <1> + cps : 10.0 <2> + ipg : 10000 <3> + rtt : 10000 <4> + w : 1 <5> + server_addr : "48.0.0.7" <6> + one_app_server : true <7> + +---- +<1> The name of the template pcap file. It can be relative to the t-rex-64 image or absolute path. The pcap file can include one flow. (Exception: in case of plug-ins). +<2> Connection per second for m==1 +<3> If the global section of the YAML file does not include `cap_ipg : true`, this line sets the inter-packet gap in microseconds. +<4> Should be set to the same value as ipg (microseconds). +<5> Default value: w=1. This indicates to the IP generator how to generate the flows. If w=2, two flows from the same template will be generated in a burst (more for HTTP that has burst of flows). +<6> If `one_app_server` is set to true, then all templates will use the same server. +<7> If the same server address is required, set this value to true. + + + +=== Configuration YAML anchor:trex_config[] + +The configuration file, in YAML format, configures TRex behavior, including: + + +- MAC address for each port (source and destination) +- Masking interfaces (usually for 1Gb/Sec TRex) to ensure that TRex does not take the management ports as traffic ports. +- Changing the zmq/telnet TCP port. + +==== Basic Configuration + +Copy/install the configuration file to `/etc/trex_cfg.yaml`. +TRex loads it automatically at startup. You still can override options with the command line option switch `--cfg [file]` in the CLI +Configuration file examples can be found in the `$ROOT/cfg` folder + + +[source,python] +---- + - port_limit : 2 <1> + version : 2 <2> + interfaces : ["03:00.0","03:00.1"] <3> + enable_zmq_pub : true <4> + zmq_pub_port : 4500 <5> + prefix : setup1 <6> + limit_memory : 1024 <7> + c : 4 <8> + port_bandwidth_gb : 10 <9> + port_info : # set eh mac addr + - dest_mac : [0x1,0x0,0x0,0x1,0x0,0x00] # port 0 <10> + src_mac : [0x2,0x0,0x0,0x2,0x0,0x00] + - dest_mac : [0x3,0x0,0x0,0x3,0x0,0x00] # port 1 + src_mac : [0x4,0x0,0x0,0x4,0x0,0x00] + - dest_mac : [0x5,0x0,0x0,0x5,0x0,0x00] # port 2 + src_mac : [0x6,0x0,0x0,0x6,0x0,0x00] + - dest_mac : [0x7,0x0,0x0,0x7,0x0,0x01] # port 3 + src_mac : [0x0,0x0,0x0,0x8,0x0,0x02] + - dest_mac : [0x0,0x0,0x0,0x9,0x0,0x03] # port 4 +---- +<1> The number of ports, should be equal to the number of interfaces in 3) +<2> Must be set to 2 +<3> Interface that should be used. used `sudo ./dpdk_setup_ports.py --show` +<4> Enable the ZMQ publisher for stats data. +<5> ZMQ port number. + +<6> The name of the setup should be distinct ( DPDK --file-prefix ) +<7> DPDK -m limit the packet memory +<8> Number of threads per dual interface ( like -c CLI option ) +<9> The bandwidth of each interface in Gb/sec. In this example we have 10Gb/sec interfaces. for VM put 1. it used to tune the amount of memory allocated by TRex. +<10> MAC address per port - source and destination. + + +To find out what the interfaces ids, perform the following: + +[source,bash] +---- + $>sudo ./dpdk_setup_ports.py --show + + Network devices using DPDK-compatible driver + ============================================ + + Network devices using kernel driver + =================================== + 0000:02:00.0 '82545EM Gigabit Ethernet Controller (Copper)' if=eth2 drv=e1000 unused=igb_uio *Active* + 0000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv= unused=ixgb #<1> + 0000:03:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv= unused=ixgb #<2> + 0000:13:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv= unused=ixgb #<3> + 0000:13:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv= unused=ixgb #<4> + + Other network devices + ===================== + <none> +---- +<1> TRex interface #1 before unbinding +<2> TRex interface #2 before unbinding +<3> TRex interface #3 before unbinding +<4> TRex interface #4 before unbinding + + +minimum configuration file is: + +[source,bash] +---- +<none> +- port_limit : 4 + version : 2 #<1> + interfaces : ["03:00.0","03:00.1","13:00.1","13:00.0"] #<2> +---- +<1> must add version 2 to the configuration file +<2> The list of interfaces from `sudo ./dpdk_setup_ports.py --show` + + +==== Memory section configuration + +The memory section is optional. It is used when there is a need to tune the amount of memory used by packet manager + +[source,python] +---- + - port_limit : 2 + version : 2 + interfaces : ["03:00.0","03:00.1"] + memory : <1> + mbuf_64 : 16380 <2> + mbuf_128 : 8190 + mbuf_256 : 8190 + mbuf_512 : 8190 + mbuf_1024 : 8190 + mbuf_2048 : 4096 + traffic_mbuf_64 : 16380 <3> + traffic_mbuf_128 : 8190 + traffic_mbuf_256 : 8190 + traffic_mbuf_512 : 8190 + traffic_mbuf_1024 : 8190 + traffic_mbuf_2048 : 4096 + dp_flows : 1048576 <4> + global_flows : 10240 <5> +---- +<1> Memory section +<2> Per dual interfaces number of buffers - buffer for real time traffic generation +<3> Traffic buffer - when you have many template only this section should be enlarge +<4> number of TRex flows needed +<5> reserved + + +==== Platform section configuration + +The platform section is optional. It is used to tune the performance and allocate the cores to the right NUMA +a configuration file now has the folowing struct to support multi instance + +[source,python] +---- +- version : 2 + interfaces : ["03:00.0","03:00.1"] + port_limit : 2 + enable_zmq_pub : true # enable publisher for stats data + zmq_pub_port : 4507 + prefix : setup1 <1> + limit_memory : 1024 <2> + c : 4 <3> + port_bandwidth_gb : 10 <4> + platform : <5> + master_thread_id : 0 <6> + latency_thread_id : 5 <7> + dual_if : + - socket : 0 <8> + threads : [1,2,3,4] <9> +---- +<1> The name of the setup should be distinct ( DPDK --file-prefix ) +<2> DPDK -m +<3> Number of threads per dual interface ( like -c CLI option ) +<4> The bandwidth of each interface in Gb/sec. In this example we have 10Gb/sec interfaces. for VM put 1. it used to tune the amount of memory allocated by TRex. +<5> the platform section +<6> The thread_id for control +<7> The thread_id for latency if used +<8> Socket of the dual interfaces, in this example of 03:00.0 and 03:00.1, memory should be local to the interface +<9> Thread to be used, should be local to the NIC + + + +=== Command line options anchor:cml-line[] + +*-f=TRAFIC_YAML_FILE*:: + Traffic YAML configuration file. + +*-c=CORES*:: + Number of cores. Use 4 for TRex 40Gb/sec. Monitor the CPU% of TRex - it should be ~50%. + +*-l=HZ*:: + Run the latency daemon in this Hz rate. Example: -l 1000 runs 1000 pkt/sec from each interface. A value of zero (0) disables the latency check. + +*-d=DURATION*:: + Duration of the test (sec), Default: 0 + +*-m=MUL*:: + Factor for bandwidth (multiply the CPS of each template by this value). + +*--ipv6*:: + Convert template to IPv6 mode. + +*--learn*:: + Learn the dynamic NAT translation and ALG. + +*--learn-verify*:: + Learn the translation. This feature is intended for verification of the mechanism in cases where there is no NAT. + +*-p*:: + Flow-flip. Sends all flow packets from the same interface. This can solve the flow order. Does not work with any router configuration. + +*-e*:: + same as `-p` but comply to the direction rules and replace source/destination IPs. it might not be good for NBAR as it is expected clients ip to be sent from same direction. + +//TBD: The last 2 sentences (flow order, router configuration) are unclear. + + +*--lm=MASK*:: + Latency mask. Use this to verify port connectivity. Possible values: 0x1 (only port 0 will send traffic), 0x2 (only port 1 will send traffic). + +*--lo*:: + Latency test. + +*--limit-ports=PORTS*:: + Limit number of ports. Configure this in the --cfg file. Possible values (number of ports): 2, 4, 6, 8. (Default: 4) + +*--nc*:: + If set, will terminate exacly at the end of the duration. This provides a faster, more accurate TRex termination. In default it wait for all the flow to terminate gracefully. In case of a very long flow the termination might be prolong. + +*-pm=MULTIFLIER*:: + Platform factor. If the setup includes a splitter, you can multiply the total results by this factor. Example: --pm 2.0 will multiply all bps results by this factor. + +*-pubd*:: + Disable ZMQ monitor's publishers. + +*-1g*:: + Deprecated. Configure TRex to 1G. Configure this in the --cfg file. + +*-k=KSEC*:: + Run a latency test before starting the test. TRex will wait for x sec before and after sending latency packets at startup. + +*--cfg=platform_yaml*:: + Load and configure platform using this file. See example file: cfg/cfg_examplexx.yaml + This file is used to configure/mask interfaces, cores, affinity, and MAC addresses. + You can use the example file by copying it to: /etc/trex_cfg.yaml + + +*-v=VERBOSE*:: + Verbose mode (works only on the debug image! ) + 1 Show only stats. + 2 Run preview. Does not write to file. + 3 Run preview and write to stats file. + Note: When using verbose mode, it is not necessary to add an output file. + Caution: Operating in verbose mode can generate very large files (terabytes). Use with caution, only on a local drive. + + +*--rx-check=SAMPLE_RATE*:: + Enable Rx check module. Using this each thread samples flows (1/sample) and checks order, latency, and additional statistics. + Note: This feature operates as an additional thread. + +*--hops=HOPES*:: + Number of hops in the setup (default is one hop). Relevant only if the Rx check is enabled. + +*--iom=MODE*:: + I/O mode for interactive mode. Possible values: 0 (silent), 1 (normal), 2 (short) + +*--no-flow-control*:: + Prevents TRex from changing flow control. In default TRex operation, flow control is disabled at startup. + +*--mac-spread*:: + Spread the destination mac by this this factor. e.g 2 will generate the traffic to 2 devices DEST-MAC ,DEST-MAC+1. The maximum is up to 128 devices. + + +ifndef::backend-docbook[] + + +endif::backend-docbook[] + +== Appendix + +=== Simulator + +The TRex simulator is a linux application that can process on any Linux CEL (it can run on TRex itself). +you can create create output pcap file from input of traffic YAML. + +==== Simulator + + +[source,bash] +---- + +$./bp-sim-64-debug -f avl/sfr_delay_10_1g.yaml -v 1 + + -- loading cap file avl/delay_10_http_get_0.pcap + -- loading cap file avl/delay_10_http_post_0.pcap + -- loading cap file avl/delay_10_https_0.pcap + -- loading cap file avl/delay_10_http_browsing_0.pcap + -- loading cap file avl/delay_10_exchange_0.pcap + -- loading cap file avl/delay_10_mail_pop_0.pcap + -- loading cap file avl/delay_10_mail_pop_1.pcap + -- loading cap file avl/delay_10_mail_pop_2.pcap + -- loading cap file avl/delay_10_oracle_0.pcap + -- loading cap file avl/delay_10_rtp_160k_full.pcap + -- loading cap file avl/delay_10_rtp_250k_full.pcap + -- loading cap file avl/delay_10_smtp_0.pcap + -- loading cap file avl/delay_10_smtp_1.pcap + -- loading cap file avl/delay_10_smtp_2.pcap + -- loading cap file avl/delay_10_video_call_0.pcap + -- loading cap file avl/delay_10_sip_video_call_full.pcap + -- loading cap file avl/delay_10_citrix_0.pcap + -- loading cap file avl/delay_10_dns_0.pcap + id,name , tps, cps,f-pkts,f-bytes, duration, Mb/sec, MB/sec, c-flows, PPS,total-Mbytes-duration,errors,flows #<2> + 00, avl/delay_10_http_get_0.pcap ,404.52,404.52, 44 , 37830 , 0.17 , 122.42 , 15.30 , 67 , 17799 , 2 , 0 , 1 + 01, avl/delay_10_http_post_0.pcap ,404.52,404.52, 54 , 48468 , 0.21 , 156.85 , 19.61 , 85 , 21844 , 2 , 0 , 1 + 02, avl/delay_10_https_0.pcap ,130.87,130.87, 96 , 91619 , 0.22 , 95.92 , 11.99 , 29 , 12564 , 1 , 0 , 1 + 03, avl/delay_10_http_browsing_0.pcap ,709.89,709.89, 37 , 34425 , 0.13 , 195.50 , 24.44 , 94 , 26266 , 2 , 0 , 1 + 04, avl/delay_10_exchange_0.pcap ,253.81,253.81, 43 , 9848 , 1.57 , 20.00 , 2.50 , 400 , 10914 , 0 , 0 , 1 + 05, avl/delay_10_mail_pop_0.pcap ,4.76,4.76, 20 , 5603 , 0.17 , 0.21 , 0.03 , 1 , 95 , 0 , 0 , 1 + 06, avl/delay_10_mail_pop_1.pcap ,4.76,4.76, 114 , 101517 , 0.25 , 3.86 , 0.48 , 1 , 543 , 0 , 0 , 1 + 07, avl/delay_10_mail_pop_2.pcap ,4.76,4.76, 30 , 15630 , 0.19 , 0.60 , 0.07 , 1 , 143 , 0 , 0 , 1 + 08, avl/delay_10_oracle_0.pcap ,79.32,79.32, 302 , 56131 , 6.86 , 35.62 , 4.45 , 544 , 23954 , 0 , 0 , 1 + 09, avl/delay_10_rtp_160k_full.pcap ,2.78,8.33, 1354 , 1232757 , 61.24 , 27.38 , 3.42 , 170 , 3759 , 0 , 0 , 3 + 10, avl/delay_10_rtp_250k_full.pcap ,1.98,5.95, 2069 , 1922000 , 61.38 , 30.48 , 3.81 , 122 , 4101 , 0 , 0 , 3 + 11, avl/delay_10_smtp_0.pcap ,7.34,7.34, 22 , 5618 , 0.19 , 0.33 , 0.04 , 1 , 161 , 0 , 0 , 1 + 12, avl/delay_10_smtp_1.pcap ,7.34,7.34, 35 , 18344 , 0.21 , 1.08 , 0.13 , 2 , 257 , 0 , 0 , 1 + 13, avl/delay_10_smtp_2.pcap ,7.34,7.34, 110 , 96544 , 0.27 , 5.67 , 0.71 , 2 , 807 , 0 , 0 , 1 + 14, avl/delay_10_video_call_0.pcap ,11.90,11.90, 2325 , 2532577 , 36.56 , 241.05 , 30.13 , 435 , 27662 , 3 , 0 , 1 + 15, avl/delay_10_sip_video_call_full.pcap ,29.35,58.69, 1651 , 120315 , 24.56 , 28.25 , 3.53 , 721 , 48452 , 0 , 0 , 2 + 16, avl/delay_10_citrix_0.pcap ,43.62,43.62, 272 , 84553 , 6.23 , 29.51 , 3.69 , 272 , 11866 , 0 , 0 , 1 + 17, avl/delay_10_dns_0.pcap ,1975.02,1975.02, 2 , 162 , 0.01 , 2.56 , 0.32 , 22 , 3950 , 0 , 0 , 1 + + 00, sum ,4083.86,93928.84, 8580 , 6413941 , 0.00 , 997.28 , 124.66 , 2966 , 215136 , 12 , 0 , 23 + Memory usage + size_64 : 1687 + size_128 : 222 + size_256 : 798 + size_512 : 1028 + size_1024 : 86 + size_2048 : 4086 + Total : 8.89 Mbytes 159% util #<1> + +---- +<1> the memory usage of the templates +<2> CSV for all the templates + diff --git a/trex_book_basic.asciidoc b/trex_book_basic.asciidoc new file mode 100755 index 00000000..d64cea43 --- /dev/null +++ b/trex_book_basic.asciidoc @@ -0,0 +1,3308 @@ + +== Basic usage + +=== DNS basic example + +The following is a simple example helpful for understanding how TRex works. The example uses the TRex simulator. +This simulator can be run on any Cisco Linux including on the TRex itself. +TRex simulates clients and servers and generates traffic based on the pcap files provided. + +.Clients/Servers +image:images/trex_model.png[title="generator"] + +The following is an example YAML-format traffic configuration file (cap2/dns_test.yaml), with explanatory notes. + +[source,python] +---- +$more cap2/dns_test.yaml +- duration : 10.0 + generator : + distribution : "seq" + clients_start : "16.0.0.1" <1> + clients_end : "16.0.0.255" + servers_start : "48.0.0.1" <2> + servers_end : "48.0.0.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 1 + udp_aging : 1 + mac : [0x00,0x00,0x00,0x01,0x00,0x00] + cap_info : + - name: cap2/dns.pcap <3> + cps : 1.0 <4> + ipg : 10000 <5> + rtt : 10000 <6> + w : 1 +---- +<1> Range of clients (IPv4 format). +<2> Range of servers (IPv4 format). +<3> pcap file, which includes the DNS cap file that will be used as a template. +<4> Number of connections per second to generate. In the example, 1.0 means 1 connection per secod. +<5> Inter-packet gap (microseconds). 10,000 = 10 msec. +<6> Should be the same as ipg. + +.DNS template file +image:images/dns_wireshark.png[title="generator"] + + +The DNS template file includes: + +1. *One* flow +2. Two packets +3. First packet: from the initiator (client -> server) +4. Second packet: response (server -> client) + +TRex replaces the client_ip, client_port, and server_ip. The server_port will be remain the same. + + +[source,bash] +---- +$./bp-sim-32-debug -f cap2/dns.yaml -o my.erf -v 3 + -- loading cap file cap2/dns.pcap + id,name , tps, cps,f-pkts,f-bytes, duration, Mb/sec, MB/sec, #<1> + 00, cap2/dns.pcap ,1.00,1.00, 2 , 170 , 0.02 , 0.00 , 0.00 , + 00, sum ,1.00,1.00, 2 , 170 , 0.00 , 0.00 , 0.00 , + + Generating erf file ... +pkt_id,time,fid,pkt_info,pkt,len,type,is_init,is_last,type,thread_id,src_ip,dest_ip,src_port #<2> + 1 ,0.010000,1,0x9055598,1,77,0,1,0,0,0,10000001,30000001,1024 + 2 ,0.020000,1,0x9054760,2,93,0,0,1,0,0,10000001,30000001,1024 + 3 ,2.010000,2,0x9055598,1,77,0,1,0,0,0,10000002,30000002,1024 + 4 ,2.020000,2,0x9054760,2,93,0,0,1,0,0,10000002,30000002,1024 + 5 ,3.010000,3,0x9055598,1,77,0,1,0,0,0,10000003,30000003,1024 + 6 ,3.020000,3,0x9054760,2,93,0,0,1,0,0,10000003,30000003,1024 + 7 ,4.010000,4,0x9055598,1,77,0,1,0,0,0,10000004,30000004,1024 + 8 ,4.020000,4,0x9054760,2,93,0,0,1,0,0,10000004,30000004,1024 + 9 ,5.010000,5,0x9055598,1,77,0,1,0,0,0,10000005,30000005,1024 + 10 ,5.020000,5,0x9054760,2,93,0,0,1,0,0,10000005,30000005,1024 + 11 ,6.010000,6,0x9055598,1,77,0,1,0,0,0,10000006,30000006,1024 + 12 ,6.020000,6,0x9054760,2,93,0,0,1,0,0,10000006,30000006,1024 + 13 ,7.010000,7,0x9055598,1,77,0,1,0,0,0,10000007,30000007,1024 + 14 ,7.020000,7,0x9054760,2,93,0,0,1,0,0,10000007,30000007,1024 + 15 ,8.010000,8,0x9055598,1,77,0,1,0,0,0,10000008,30000008,1024 + 16 ,8.020000,8,0x9054760,2,93,0,0,1,0,0,10000008,30000008,1024 + 17 ,9.010000,9,0x9055598,1,77,0,1,0,0,0,10000009,30000009,1024 + 18 ,9.020000,9,0x9054760,2,93,0,0,1,0,0,10000009,30000009,1024 + 19 ,10.010000,a,0x9055598,1,77,0,1,0,0,0,1000000a,3000000a,1024 + 20 ,10.020000,a,0x9054760,2,93,0,0,1,0,0,1000000a,3000000a,1024 + +file stats +================= + m_total_bytes : 1.66 Kbytes + m_total_pkt : 20.00 pkt + m_total_open_flows : 10.00 flows + m_total_pkt : 20 + m_total_open_flows : 10 + m_total_close_flows : 10 + m_total_bytes : 1700 +---- +<1> Global statistics on the templates given. cps=connection per second. tps is template per second. they might be different in case of plugins where one template includes more than one flow. For example RTP flow in SFR profile (avl/delay_10_rtp_160k_full.pcap) +<2> Generator output. + + +[source,bash] +---- +$wireshark my.erf +---- +gives +//TBD: Not sure what the output looks like here, with this line showing only "gives" + +.TRex generated output file +image:images/dns_trex_run.png[title="generator"] + +As the output file shows... + +- TRex generates a new flow every 1 sec. +- Client IP values are taken from client IP pool . +- Servers IP values are taken from server IP pool . +- IPG (iter packet gap) values are taken from the configuration file (10 msec). + + +[NOTE] +===================================================================== +In basic usage, TRex does not wait for an initiator packet to be received. The response packet will be triggered based only on timeout (IPG in this example). +In advanced scenarios (for example, NAT), The first packet of the flow can process by TRex software and initiate the response packet only when a packet is received. +Consequently, it is necessary to *process* the template pcap file offline and ensure that there is enough round-trip delay (RTT) between client and server packets. +One approach is to record the flow with a Pagent that creats RTT (10 msec RTT in the example), recording the traffic at some distance from both the client and server (not close to either side). +This ensures sufficient delay that packets from each side will arrive without delay in the DUT. TRex-dev will work on an offline tool that will make it even simpler. +Another approach is to change the `yaml` `ipg` field to a high enough value (bigger than 10msec ). +===================================================================== + +Converting the simulator text results in a table similar to the following: + +.DNS example formatted results +[format="csv",cols="1^,2^,1^,1^,2^,1^,2^,1^", options="header"] +|================= + pkt,time sec,fid,flow-pkt-id,client_ip,client_port,server_ip ,direction + 1 , 0.010000 , 1 , 1 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 2 , 0.020000 , 1 , 2 , 16.0.0.1 , 1024 , 48.0.0.1 , <- + 3 , 2.010000 , 2 , 1 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 4 , 2.020000 , 2 , 2 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 5 , 3.010000 , 3 , 1 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 6 , 3.020000 , 3 , 2 , 16.0.0.3 , 1024 , 48.0.0.3 , <- + 7 , 4.010000 , 4 , 1 , 16.0.0.4 , 1024 , 48.0.0.4 , -> + 8 , 4.020000 , 4 , 2 , 16.0.0.4 , 1024 , 48.0.0.4 , <- + 9 , 5.010000 , 5 , 1 , 16.0.0.5 , 1024 , 48.0.0.5 , -> + 10 , 5.020000 , 5 , 2 , 16.0.0.5 , 1024 , 48.0.0.5 , <- + 11 , 6.010000 , 6 , 1 , 16.0.0.6 , 1024 , 48.0.0.6 , -> + 12 , 6.020000 , 6 , 2 , 16.0.0.6 , 1024 , 48.0.0.6 , <- + 13 , 7.010000 , 7 , 1 , 16.0.0.7 , 1024 , 48.0.0.7 , -> + 14 , 7.020000 , 7 , 2 , 16.0.0.7 , 1024 , 48.0.0.7 , <- + 15 , 8.010000 , 8 , 1 , 16.0.0.8 , 1024 , 48.0.0.8 , -> + 16 , 8.020000 , 8 , 2 , 16.0.0.8 , 1024 , 48.0.0.8 , <- + 17 , 9.010000 , 9 , 1 , 16.0.0.9 , 1024 , 48.0.0.9 , -> + 18 , 9.020000 , 9 , 2 , 16.0.0.9 , 1024 , 48.0.0.9 , <- + 19 , 10.010000 , a , 1 , 16.0.0.10 , 1024 , 48.0.0.10 , -> + 20 , 10.020000 , a , 2 , 16.0.0.10 , 1024 , 48.0.0.10 , <- +|================= + +where: +fid:: + Flow ID - different IDs for each flow. + +low-pkt-id:: + Packet ID within the flow. Numbering begins with 1. + +client_ip:: + Client IP address. + +client_port:: + Client IP port. + +server_ip:: + Server IP address. + +direction:: + Direction. "->" is client-to-server; "<-" is server-to-client. + + + +The following enlarges the CPS and reduces the duration. + +[source,python] +---- +$more cap2/dns_test.yaml +- duration : 1.0 <1> + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.255" + servers_start : "48.0.0.1" + servers_end : "48.0.0.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 1 + udp_aging : 1 + mac : [0x00,0x00,0x00,0x01,0x00,0x00] + cap_info : + - name: cap2/dns.pcap + cps : 10.0 <2> + ipg : 50000 <3> + rtt : 50000 + w : 1 +---- +<1> Duration is 1 second. +<2> CPS is 10.0. +<3> IPG is 50 msec. + + +Running this produces the following output: + +[source,bash] +---- +$./bp-sim-32-debug -f cap2/dns_test.yaml -o my.erf -v 3 +---- + +.Formated results +[format="csv",cols="1^,2^,1^,1^,1^,2^,1^,2^,1^", options="header"] +|================= +pkt,time sec,template,fid,flow-pkt-id,client_ip,client_port,server_ip ,desc + 1 , 0.010000 , 0 , 1 , 1 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 2 , 0.060000 , 0 , 1 , 2 , 16.0.0.1 , 1024 , 48.0.0.1 , <- + 3 , 0.210000 , 0 , 2 , 1 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 4 , 0.260000 , 0 , 2 , 2 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 5 , 0.310000 , 0 , 3 , 1 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 6 , 0.360000 , 0 , 3 , 2 , 16.0.0.3 , 1024 , 48.0.0.3 , <- + 7 , 0.410000 , 0 , 4 , 1 , 16.0.0.4 , 1024 , 48.0.0.4 , -> + 8 , 0.460000 , 0 , 4 , 2 , 16.0.0.4 , 1024 , 48.0.0.4 , <- + 9 , 0.510000 , 0 , 5 , 1 , 16.0.0.5 , 1024 , 48.0.0.5 , -> + 10 , 0.560000 , 0 , 5 , 2 , 16.0.0.5 , 1024 , 48.0.0.5 , <- + 11 , 0.610000 , 0 , 6 , 1 , 16.0.0.6 , 1024 , 48.0.0.6 , -> + 12 , 0.660000 , 0 , 6 , 2 , 16.0.0.6 , 1024 , 48.0.0.6 , <- + 13 , 0.710000 , 0 , 7 , 1 , 16.0.0.7 , 1024 , 48.0.0.7 , -> + 14 , 0.760000 , 0 , 7 , 2 , 16.0.0.7 , 1024 , 48.0.0.7 , <- + 15 , 0.810000 , 0 , 8 , 1 , 16.0.0.8 , 1024 , 48.0.0.8 , -> + 16 , 0.860000 , 0 , 8 , 2 , 16.0.0.8 , 1024 , 48.0.0.8 , <- + 17 , 0.910000 , 0 , 9 , 1 , 16.0.0.9 , 1024 , 48.0.0.9 , -> + 18 , 0.960000 , 0 , 9 , 2 , 16.0.0.9 , 1024 , 48.0.0.9 , <- + 19 , 1.010000 , 0 , a , 1 , 16.0.0.10 , 1024 , 48.0.0.10 , -> + 20 , 1.060000 , 0 , a , 2 , 16.0.0.10 , 1024 , 48.0.0.10 , <- +|================= + +Use the following to display the output as a chart, with: +x axis: time (seconds) +y axis: flow ID +The output indicates that there are 10 flows in 1 second, as expected, and the IPG is 50 msec + +//TBD: not sure what the "+ +" means ==> [hh] Ascii Doc break page + +ifndef::backend-docbook[] ++++++++++++++++++++++++++++++++++ +<div id="chart1" style="font : 10px sans-serif"; ></div> + +<script> +var dns_data=[ +[ 0.010000 , 1 , 0 , 1 ], +[ 0.060000 , 1 , 0 , 2 ], +[ 0.210000 , 2 , 0 , 1 ], +[ 0.260000 , 2 , 0 , 2 ], +[ 0.310000 , 3 , 0 , 1 ], +[ 0.360000 , 3 , 0 , 2 ], +[ 0.410000 , 4 , 0 , 1 ], +[ 0.460000 , 4 , 0 , 2 ], +[ 0.510000 , 5 , 0 , 1 ], +[ 0.560000 , 5 , 0 , 2 ], +[ 0.610000 , 6 , 0 , 1 ], +[ 0.660000 , 6 , 0 , 2 ], +[ 0.710000 , 7 , 0 , 1 ], +[ 0.760000 , 7 , 0 , 2 ], +[ 0.810000 , 8 , 0 , 1 ], +[ 0.860000 , 8 , 0 , 2 ], +[ 0.910000 , 9 , 0 , 1 ], +[ 0.960000 , 9 , 0 , 2 ], +[ 1.010000 , 10 , 0 , 1 ], +[ 1.060000 , 10 , 0 , 2 ], +]; +chart("#chart1",dns_data,["dns"],"time-sec","flow-id"); +</script> ++++++++++++++++++++++++++++++++++ +endif::backend-docbook[] + +[NOTE] +===================================================================== +Note the gap in the second flow generation. This is an expected schedular artifact and does not have an effect. +===================================================================== + +=== DNS, take flow IPG from pcap file + +In the following example the IPG is taken from the IPG itself. + +[source,python] +---- +- duration : 1.0 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.255" + servers_start : "48.0.0.1" + servers_end : "48.0.0.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 0 + udp_aging : 0 + mac : [0x00,0x00,0x00,0x01,0x00,0x00] + cap_ipg : true <1> + #cap_ipg_min : 30 + #cap_override_ipg : 200 + cap_info : + - name: cap2/dns.pcap + cps : 10.0 + ipg : 10000 + rtt : 10000 + w : 1 +---- +<1> IPG is taken from pcap. + + +.dns ipg from pcap file +[format="csv",cols="1^,2^,1^,1^,1^,2^,1^,2^,1^", options="header"] +|================= +pkt,time sec,template,fid,flow-pkt-id,client_ip,client_port,server_ip ,desc + 1 , 0.010000 , 0 , 1 , 1 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 2 , 0.030944 , 0 , 1 , 2 , 16.0.0.1 , 1024 , 48.0.0.1 , <- + 3 , 0.210000 , 0 , 2 , 1 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 4 , 0.230944 , 0 , 2 , 2 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 5 , 0.310000 , 0 , 3 , 1 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 6 , 0.330944 , 0 , 3 , 2 , 16.0.0.3 , 1024 , 48.0.0.3 , <- + 7 , 0.410000 , 0 , 4 , 1 , 16.0.0.4 , 1024 , 48.0.0.4 , -> + 8 , 0.430944 , 0 , 4 , 2 , 16.0.0.4 , 1024 , 48.0.0.4 , <- + 9 , 0.510000 , 0 , 5 , 1 , 16.0.0.5 , 1024 , 48.0.0.5 , -> + 10 , 0.530944 , 0 , 5 , 2 , 16.0.0.5 , 1024 , 48.0.0.5 , <- + 11 , 0.610000 , 0 , 6 , 1 , 16.0.0.6 , 1024 , 48.0.0.6 , -> + 12 , 0.630944 , 0 , 6 , 2 , 16.0.0.6 , 1024 , 48.0.0.6 , <- + 13 , 0.710000 , 0 , 7 , 1 , 16.0.0.7 , 1024 , 48.0.0.7 , -> + 14 , 0.730944 , 0 , 7 , 2 , 16.0.0.7 , 1024 , 48.0.0.7 , <- + 15 , 0.810000 , 0 , 8 , 1 , 16.0.0.8 , 1024 , 48.0.0.8 , -> + 16 , 0.830944 , 0 , 8 , 2 , 16.0.0.8 , 1024 , 48.0.0.8 , <- + 17 , 0.910000 , 0 , 9 , 1 , 16.0.0.9 , 1024 , 48.0.0.9 , -> + 18 , 0.930944 , 0 , 9 , 2 , 16.0.0.9 , 1024 , 48.0.0.9 , <- + 19 , 1.010000 , 0 , a , 1 , 16.0.0.10 , 1024 , 48.0.0.10 , -> + 20 , 1.030944 , 0 , a , 2 , 16.0.0.10 , 1024 , 48.0.0.10 , <- +|================= + +In this example, the IPG was taken from the pcap file, which is closer to 20 msec and not 50 msec (taken from the configuration file). + +[source,python] +---- + #cap_ipg_min : 30 <1> + #cap_override_ipg : 200 <2> +---- +<1> Sets the minimum IPG (microseconds) which should be override : ( if (pkt_ipg<cap_ipg_min) { pkt_ipg = cap_override_ipg } ) +<2> Value to override (microseconds). + + +ifndef::backend-docbook[] ++++++++++++++++++++++++++++++++++ +<div id="chart2" style="font : 10px sans-serif"; ></div> + +<script> +var chart2_data=[ +[ 0.010000 , 1 , 0 , 1 ], +[ 0.030944 , 1 , 0 , 2 ], +[ 0.210000 , 2 , 0 , 1 ], +[ 0.230944 , 2 , 0 , 2 ], +[ 0.310000 , 3 , 0 , 1 ], +[ 0.330944 , 3 , 0 , 2 ], +[ 0.410000 , 4 , 0 , 1 ], +[ 0.430944 , 4 , 0 , 2 ], +[ 0.510000 , 5 , 0 , 1 ], +[ 0.530944 , 5 , 0 , 2 ], +[ 0.610000 , 6 , 0 , 1 ], +[ 0.630944 , 6 , 0 , 2 ], +[ 0.710000 , 7 , 0 , 1 ], +[ 0.730944 , 7 , 0 , 2 ], +[ 0.810000 , 8 , 0 , 1 ], +[ 0.830944 , 8 , 0 , 2 ], +[ 0.910000 , 9 , 0 , 1 ], +[ 0.930944 , 9 , 0 , 2 ], +[ 1.010000 , 10 , 0 , 1 ], +[ 1.030944 , 10 , 0 , 2 ], +]; +chart("#chart2",chart2_data,["dns"],"time-sec","flow-id"); +</script> ++++++++++++++++++++++++++++++++++ +endif::backend-docbook[] + + +=== DNS, Set one server ip + +In this example the server IP is taken from the template. + +[source,python] +---- +- duration : 10.0 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.1.255" + servers_start : "48.0.0.1" + servers_end : "48.0.0.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 1 + udp_aging : 1 + mac : [0x00,0x00,0x00,0x01,0x00,0x00] + cap_ipg : true + #cap_ipg_min : 30 + #cap_override_ipg : 200 + cap_info : + - name: cap2/dns.pcap + cps : 1.0 + ipg : 10000 + rtt : 10000 + server_addr : "48.0.0.7" <1> + one_app_server : true <2> + w : 1 +---- +<1> All templates will use the same server. +<2> Must be set to "true". + + +.dns ipg from pcap file +[format="csv",cols="1^,2^,1^,1^,2^,1^,2^,1^", options="header"] +|================= +pkt,time sec,fid,flow-pkt-id,client_ip,client_port,server_ip ,desc + 1 , 0.010000 , 1 , 1 , 16.0.0.1 , 1024 , 48.0.0.7 , -> + 2 , 0.030944 , 1 , 2 , 16.0.0.1 , 1024 , 48.0.0.7 , <- + 3 , 2.010000 , 2 , 1 , 16.0.0.2 , 1024 , 48.0.0.7 , -> + 4 , 2.030944 , 2 , 2 , 16.0.0.2 , 1024 , 48.0.0.7 , <- + 5 , 3.010000 , 3 , 1 , 16.0.0.3 , 1024 , 48.0.0.7 , -> + 6 , 3.030944 , 3 , 2 , 16.0.0.3 , 1024 , 48.0.0.7 , <- + 7 , 4.010000 , 4 , 1 , 16.0.0.4 , 1024 , 48.0.0.7 , -> + 8 , 4.030944 , 4 , 2 , 16.0.0.4 , 1024 , 48.0.0.7 , <- + 9 , 5.010000 , 5 , 1 , 16.0.0.5 , 1024 , 48.0.0.7 , -> + 10 , 5.030944 , 5 , 2 , 16.0.0.5 , 1024 , 48.0.0.7 , <- + 11 , 6.010000 , 6 , 1 , 16.0.0.6 , 1024 , 48.0.0.7 , -> + 12 , 6.030944 , 6 , 2 , 16.0.0.6 , 1024 , 48.0.0.7 , <- + 13 , 7.010000 , 7 , 1 , 16.0.0.7 , 1024 , 48.0.0.7 , -> + 14 , 7.030944 , 7 , 2 , 16.0.0.7 , 1024 , 48.0.0.7 , <- + 15 , 8.010000 , 8 , 1 , 16.0.0.8 , 1024 , 48.0.0.7 , -> + 16 , 8.030944 , 8 , 2 , 16.0.0.8 , 1024 , 48.0.0.7 , <- + 17 , 9.010000 , 9 , 1 , 16.0.0.9 , 1024 , 48.0.0.7 , -> + 18 , 9.030944 , 9 , 2 , 16.0.0.9 , 1024 , 48.0.0.7 , <- + 19 , 10.010000 , a , 1 , 16.0.0.10 , 1024 , 48.0.0.7 , -> + 20 , 10.030944 , a , 2 , 16.0.0.10 , 1024 , 48.0.0.7 , <- +|================= + + +=== DNS, Reduce the number of clients +//TBD: clarify + +[source,python] +---- +- duration : 10.0 + generator : + distribution : "seq" + clients_start : "16.0.0.1" <1> + clients_end : "16.0.0.1" + servers_start : "48.0.0.1" + servers_end : "48.0.0.3" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 1 + udp_aging : 1 + mac : [0x00,0x00,0x00,0x01,0x00,0x00] + cap_ipg : true + #cap_ipg_min : 30 + #cap_override_ipg : 200 + cap_info : + - name: cap2/dns.pcap + cps : 1.0 + ipg : 10000 + rtt : 10000 + w : 1 +---- +<1> Only one client. + + +.dns ipg from pcap file +[format="csv",cols="1^,2^,1^,1^,2^,1^,2^,1^", options="header"] +|================= +pkt,time sec,fid,flow-pkt-id,client_ip,client_port,server_ip ,desc + 1 , 0.010000 , 1 , 1 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 2 , 0.030944 , 1 , 2 , 16.0.0.1 , 1024 , 48.0.0.1 , <- + 3 , 2.010000 , 2 , 1 , 16.0.0.1 , 1025 , 48.0.0.2 , -> + 4 , 2.030944 , 2 , 2 , 16.0.0.1 , 1025 , 48.0.0.2 , <- + 5 , 3.010000 , 3 , 1 , 16.0.0.1 , 1026 , 48.0.0.3 , -> + 6 , 3.030944 , 3 , 2 , 16.0.0.1 , 1026 , 48.0.0.3 , <- + 7 , 4.010000 , 4 , 1 , 16.0.0.1 , 1027 , 48.0.0.4 , -> + 8 , 4.030944 , 4 , 2 , 16.0.0.1 , 1027 , 48.0.0.4 , <- + 9 , 5.010000 , 5 , 1 , 16.0.0.1 , 1028 , 48.0.0.5 , -> + 10 , 5.030944 , 5 , 2 , 16.0.0.1 , 1028 , 48.0.0.5 , <- + 11 , 6.010000 , 6 , 1 , 16.0.0.1 , 1029 , 48.0.0.6 , -> + 12 , 6.030944 , 6 , 2 , 16.0.0.1 , 1029 , 48.0.0.6 , <- + 13 , 7.010000 , 7 , 1 , 16.0.0.1 , 1030 , 48.0.0.7 , -> + 14 , 7.030944 , 7 , 2 , 16.0.0.1 , 1030 , 48.0.0.7 , <- + 15 , 8.010000 , 8 , 1 , 16.0.0.1 , 1031 , 48.0.0.8 , -> + 16 , 8.030944 , 8 , 2 , 16.0.0.1 , 1031 , 48.0.0.8 , <- + 17 , 9.010000 , 9 , 1 , 16.0.0.1 , 1032 , 48.0.0.9 , -> + 18 , 9.030944 , 9 , 2 , 16.0.0.1 , 1032 , 48.0.0.9 , <- + 19 , 10.010000 , a , 1 , 16.0.0.1 , 1033 , 48.0.0.10 , -> + 20 , 10.030944 , a , 2 , 16.0.0.1 , 1033 , 48.0.0.10 , <- +|================= + +In this case there is only one client so only ports are used to distinc the flows +you need to be sure that you have enogth free sockets when running TRex in high rates + +[source,python] +---- + Active-flows : 0 Clients : 1 <1> Socket-util : 0.0000 % <2> + Open-flows : 1 Servers : 254 Socket : 1 Socket/Clients : 0.0 + drop-rate : 0.00 bps +---- +<1> Number of clients +<2> sockets utilization (should be lowwer than 20%, elarge the number of clients in case of an issue). + +=== DNS, W=1 + +`w` is a tunable to the IP clients/servers generator. w=1 is the default behavior. +Setting `w=2` configures a burst of two allocations from the same client. See the following example. + +[source,python] +---- +- duration : 10.0 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.10" + servers_start : "48.0.0.1" + servers_end : "48.0.0.3" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 1 + udp_aging : 1 + mac : [0x00,0x00,0x00,0x01,0x00,0x00] + cap_ipg : true + #cap_ipg_min : 30 + #cap_override_ipg : 200 + cap_info : + - name: cap2/dns.pcap + cps : 1.0 + ipg : 10000 + rtt : 10000 + w : 2 <1> +---- +<1> Two clients will be allocated from the same template. + + +.DNS ipg from pcap file +[format="csv",cols="1^,2^,1^,1^,2^,1^,2^,1^", options="header"] +|================= +pkt,time sec,fid,flow-pkt-id,client_ip,client_port,server_ip ,desc + 1 , 0.010000 , 1 , 1 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 2 , 0.030944 , 1 , 2 , 16.0.0.1 , 1024 , 48.0.0.1 , <- + 3 , 2.010000 , 2 , 1 , 16.0.0.1 , 1025 , 48.0.0.1 , -> + 4 , 2.030944 , 2 , 2 , 16.0.0.1 , 1025 , 48.0.0.1 , <- + 5 , 3.010000 , 3 , 1 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 6 , 3.030944 , 3 , 2 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 7 , 4.010000 , 4 , 1 , 16.0.0.2 , 1025 , 48.0.0.2 , -> + 8 , 4.030944 , 4 , 2 , 16.0.0.2 , 1025 , 48.0.0.2 , <- + 9 , 5.010000 , 5 , 1 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 10 , 5.030944 , 5 , 2 , 16.0.0.3 , 1024 , 48.0.0.3 , <- + 11 , 6.010000 , 6 , 1 , 16.0.0.3 , 1025 , 48.0.0.3 , -> + 12 , 6.030944 , 6 , 2 , 16.0.0.3 , 1025 , 48.0.0.3 , <- + 13 , 7.010000 , 7 , 1 , 16.0.0.4 , 1024 , 48.0.0.4 , -> + 14 , 7.030944 , 7 , 2 , 16.0.0.4 , 1024 , 48.0.0.4 , <- + 15 , 8.010000 , 8 , 1 , 16.0.0.4 , 1025 , 48.0.0.4 , -> + 16 , 8.030944 , 8 , 2 , 16.0.0.4 , 1025 , 48.0.0.4 , <- + 17 , 9.010000 , 9 , 1 , 16.0.0.5 , 1024 , 48.0.0.5 , -> + 18 , 9.030944 , 9 , 2 , 16.0.0.5 , 1024 , 48.0.0.5 , <- + 19 , 10.010000 , a , 1 , 16.0.0.5 , 1025 , 48.0.0.5 , -> + 20 , 10.030944 , a , 2 , 16.0.0.5 , 1025 , 48.0.0.5 , <- +|================= + + +=== Mixing HTTP and DNS template + +The following example combines elements of HTTP and DNS templates: + + +[source,python] +---- +- duration : 1.0 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.10" + servers_start : "48.0.0.1" + servers_end : "48.0.0.3" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 1 + udp_aging : 1 + mac : [0x00,0x00,0x00,0x01,0x00,0x00] + cap_ipg : true + cap_info : + - name: cap2/dns.pcap + cps : 10.0 <1> + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_http_browsing_0.pcap + cps : 2.0 <1> + ipg : 10000 + rtt : 10000 + w : 1 + +---- +<1> Same CPS for both templates. + +This creates the following output: + +.DNS ipg from pcap file +[format="csv",cols="1^,2^,1^,1^,1^,2^,1^,2^,1^", options="header"] +|================= +pkt,time sec,template,fid,flow-pkt-id,client_ip,client_port,server_ip ,desc + 1 , 0.010000 , 0 , 1 , 1 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 2 , 0.030944 , 0 , 1 , 2 , 16.0.0.1 , 1024 , 48.0.0.1 , <- + 3 , 0.093333 , 1 , 2 , 1 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 4 , 0.104362 , 1 , 2 , 2 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 5 , 0.115385 , 1 , 2 , 3 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 6 , 0.115394 , 1 , 2 , 4 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 7 , 0.126471 , 1 , 2 , 5 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 8 , 0.126484 , 1 , 2 , 6 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 9 , 0.137530 , 1 , 2 , 7 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 10 , 0.148609 , 1 , 2 , 8 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 11 , 0.148621 , 1 , 2 , 9 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 12 , 0.148635 , 1 , 2 , 10 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 13 , 0.159663 , 1 , 2 , 11 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 14 , 0.170750 , 1 , 2 , 12 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 15 , 0.170762 , 1 , 2 , 13 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 16 , 0.170774 , 1 , 2 , 14 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 17 , 0.176667 , 0 , 3 , 1 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 18 , 0.181805 , 1 , 2 , 15 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 19 , 0.181815 , 1 , 2 , 16 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 20 , 0.192889 , 1 , 2 , 17 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 21 , 0.192902 , 1 , 2 , 18 , 16.0.0.2 , 1024 , 48.0.0.2 , <- +|================= + +Template_id:: + 0: DNS template + 1: HTTP template + + +ifndef::backend-docbook[] ++++++++++++++++++++++++++++++++++ +<div id="chart3" style="font : 10px sans-serif"; ></div> + +<script> +var chart3_data=[ +[ 0.010000 , 1 , 0 , 1 ], +[ 0.030944 , 1 , 0 , 2 ], +[ 0.093333 , 2 , 1 , 1 ], +[ 0.104362 , 2 , 1 , 2 ], +[ 0.115385 , 2 , 1 , 3 ], +[ 0.115394 , 2 , 1 , 4 ], +[ 0.126471 , 2 , 1 , 5 ], +[ 0.126484 , 2 , 1 , 6 ], +[ 0.137530 , 2 , 1 , 7 ], +[ 0.148609 , 2 , 1 , 8 ], +[ 0.148621 , 2 , 1 , 9 ], +[ 0.148635 , 2 , 1 , 10 ], +[ 0.159663 , 2 , 1 , 11 ], +[ 0.170750 , 2 , 1 , 12 ], +[ 0.170762 , 2 , 1 , 13 ], +[ 0.170774 , 2 , 1 , 14 ], +[ 0.176667 , 3 , 0 , 1 ], +[ 0.181805 , 2 , 1 , 15 ], +[ 0.181815 , 2 , 1 , 16 ], +[ 0.192889 , 2 , 1 , 17 ], +[ 0.192902 , 2 , 1 , 18 ], +[ 0.192914 , 2 , 1 , 19 ], +[ 0.192927 , 2 , 1 , 20 ], +[ 0.192939 , 2 , 1 , 21 ], +[ 0.192951 , 2 , 1 , 22 ], +[ 0.197611 , 3 , 0 , 2 ], +[ 0.203944 , 2 , 1 , 23 ], +[ 0.203950 , 2 , 1 , 24 ], +[ 0.203956 , 2 , 1 , 25 ], +[ 0.214620 , 2 , 1 , 26 ], +[ 0.214633 , 2 , 1 , 27 ], +[ 0.214645 , 2 , 1 , 28 ], +[ 0.214658 , 2 , 1 , 29 ], +[ 0.214671 , 2 , 1 , 30 ], +[ 0.214682 , 2 , 1 , 31 ], +[ 0.214695 , 2 , 1 , 32 ], +[ 0.214707 , 2 , 1 , 33 ], +[ 0.225264 , 2 , 1 , 34 ], +[ 0.225269 , 2 , 1 , 35 ], +[ 0.225274 , 2 , 1 , 36 ], +[ 0.225279 , 2 , 1 , 37 ], +[ 0.260000 , 4 , 0 , 1 ], +[ 0.280944 , 4 , 0 , 2 ], +[ 0.343333 , 5 , 0 , 1 ], +[ 0.364277 , 5 , 0 , 2 ], +[ 0.426667 , 6 , 0 , 1 ], +[ 0.447611 , 6 , 0 , 2 ], +[ 0.593333 , 7 , 0 , 1 ], +[ 0.614277 , 7 , 0 , 2 ], +[ 0.676667 , 8 , 1 , 1 ], +[ 0.687696 , 8 , 1 , 2 ], +[ 0.698719 , 8 , 1 , 3 ], +[ 0.698728 , 8 , 1 , 4 ], +[ 0.709805 , 8 , 1 , 5 ], +[ 0.709818 , 8 , 1 , 6 ], +[ 0.720864 , 8 , 1 , 7 ], +[ 0.731943 , 8 , 1 , 8 ], +[ 0.731955 , 8 , 1 , 9 ], +[ 0.731968 , 8 , 1 , 10 ], +[ 0.742997 , 8 , 1 , 11 ], +[ 0.754084 , 8 , 1 , 12 ], +[ 0.754096 , 8 , 1 , 13 ], +[ 0.754108 , 8 , 1 , 14 ], +[ 0.760000 , 9 , 0 , 1 ], +[ 0.765139 , 8 , 1 , 15 ], +[ 0.765148 , 8 , 1 , 16 ], +[ 0.776223 , 8 , 1 , 17 ], +[ 0.776236 , 8 , 1 , 18 ], +[ 0.776248 , 8 , 1 , 19 ], +[ 0.776261 , 8 , 1 , 20 ], +[ 0.776273 , 8 , 1 , 21 ], +[ 0.776285 , 8 , 1 , 22 ], +[ 0.780944 , 9 , 0 , 2 ], +[ 0.787278 , 8 , 1 , 23 ], +[ 0.787284 , 8 , 1 , 24 ], +[ 0.787289 , 8 , 1 , 25 ], +[ 0.797954 , 8 , 1 , 26 ], +[ 0.797967 , 8 , 1 , 27 ], +[ 0.797979 , 8 , 1 , 28 ], +[ 0.797992 , 8 , 1 , 29 ], +[ 0.798004 , 8 , 1 , 30 ], +[ 0.798016 , 8 , 1 , 31 ], +[ 0.798029 , 8 , 1 , 32 ], +[ 0.798041 , 8 , 1 , 33 ], +[ 0.808598 , 8 , 1 , 34 ], +[ 0.808603 , 8 , 1 , 35 ], +[ 0.808608 , 8 , 1 , 36 ], +[ 0.808613 , 8 , 1 , 37 ], +[ 0.843333 , 10 , 0 , 1 ], +[ 0.864277 , 10 , 0 , 2 ], +[ 0.926667 , 11 , 0 , 1 ], +[ 0.947611 , 11 , 0 , 2 ], +[ 1.010000 , 12 , 0 , 1 ], +[ 1.030944 , 12 , 0 , 2 ], +]; +chart("#chart3",chart3_data,["dns","http"],"time-sec","flow-id"); +</script> ++++++++++++++++++++++++++++++++++ +endif::backend-docbook[] + +The output above illustrates two HTTP flows and ten DNS flows in 1 second, as expected. + + +=== SFR traffic YAML + +SFR traffic includes a combination of traffic templates. This traffic mix in the example below was defined by SFR France. +This SFR traffic profile is used as our traffic profile for our ASR1k/ISR-G2 benchmark. It is also possible to use EMIX instead of IMIX traffic. + +The traffic was recorded from a Spirent C100 with a Pagent that introduce 10msec delay from client and server side. + +[source,python] +---- +- duration : 0.1 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.1.255" + servers_start : "48.0.0.1" + servers_end : "48.0.20.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 0 + udp_aging : 0 + mac : [0x0,0x0,0x0,0x1,0x0,0x00] + cap_ipg : true + cap_info : + - name: avl/delay_10_http_get_0.pcap + cps : 404.52 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_http_post_0.pcap + cps : 404.52 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_https_0.pcap + cps : 130.8745 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_http_browsing_0.pcap + cps : 709.89 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_exchange_0.pcap + cps : 253.81 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_mail_pop_0.pcap + cps : 4.759 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_mail_pop_1.pcap + cps : 4.759 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_mail_pop_2.pcap + cps : 4.759 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_oracle_0.pcap + cps : 79.3178 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_rtp_160k_full.pcap + cps : 2.776 + ipg : 10000 + rtt : 10000 + w : 1 + one_app_server : false + plugin_id : 1 <2> + - name: avl/delay_10_rtp_250k_full.pcap + cps : 1.982 + ipg : 10000 + rtt : 10000 + w : 1 + one_app_server : false + plugin_id : 1 + - name: avl/delay_10_smtp_0.pcap + cps : 7.3369 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_smtp_1.pcap + cps : 7.3369 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_smtp_2.pcap + cps : 7.3369 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_video_call_0.pcap + cps : 11.8976 + ipg : 10000 + rtt : 10000 + w : 1 + one_app_server : false + - name: avl/delay_10_sip_video_call_full.pcap + cps : 29.347 + ipg : 10000 + rtt : 10000 + w : 1 + plugin_id : 2 <1> + one_app_server : false + - name: avl/delay_10_citrix_0.pcap + cps : 43.6248 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_dns_0.pcap + cps : 1975.015 + ipg : 10000 + rtt : 10000 + w : 1 + wlength : 1 +---- +<1> Plugin for SIP protocol, used to replace the IP/port in the control flow base on the data-flow. +//TBD: I'm placing your note into a TBD - (what are plugins should have a seperate chapter) +<2> Plugin for RTSP protocol used to replace the IP/port in the control flow base on the data-flow. + + + +ifndef::backend-docbook[] ++++++++++++++++++++++++++++++++++ +<div id="chart4" style="font : 10px sans-serif"; ></div> + +<script> + +var sfr_data=[ +[ 0.010000 , 1 , 0 , 1 ], +[ 0.010245 , 2 , 1 , 1 ], +[ 0.010490 , 3 , 2 , 1 ], +[ 0.010735 , 4 , 3 , 1 ], +[ 0.010979 , 5 , 4 , 1 ], +[ 0.011224 , 6 , 5 , 1 ], +[ 0.011469 , 7 , 6 , 1 ], +[ 0.011714 , 8 , 7 , 1 ], +[ 0.011959 , 9 , 8 , 1 ], +[ 0.012204 , 10 , 9 , 1 ], +[ 0.012449 , 11 , 10 , 1 ], +[ 0.012694 , 12 , 11 , 1 ], +[ 0.012938 , 13 , 12 , 1 ], +[ 0.013183 , 14 , 13 , 1 ], +[ 0.013428 , 15 , 14 , 1 ], +[ 0.013673 , 16 , 15 , 1 ], +[ 0.013918 , 17 , 16 , 1 ], +[ 0.014163 , 18 , 17 , 1 ], +[ 0.014408 , 19 , 0 , 1 ], +[ 0.014652 , 20 , 1 , 1 ], +[ 0.014897 , 21 , 3 , 1 ], +[ 0.015142 , 22 , 4 , 1 ], +[ 0.015387 , 23 , 17 , 1 ], +[ 0.015632 , 24 , 0 , 1 ], +[ 0.015877 , 25 , 1 , 1 ], +[ 0.016122 , 26 , 3 , 1 ], +[ 0.016367 , 27 , 17 , 1 ], +[ 0.016611 , 28 , 3 , 1 ], +[ 0.016856 , 29 , 17 , 1 ], +[ 0.017101 , 30 , 3 , 1 ], +[ 0.017346 , 31 , 17 , 1 ], +[ 0.017591 , 32 , 0 , 1 ], +[ 0.017836 , 33 , 1 , 1 ], +[ 0.018081 , 34 , 3 , 1 ], +[ 0.018325 , 35 , 17 , 1 ], +[ 0.018456 , 15 , 14 , 2 ], +[ 0.018570 , 36 , 2 , 1 ], +[ 0.018815 , 37 , 17 , 1 ], +[ 0.019060 , 38 , 4 , 1 ], +[ 0.019305 , 39 , 17 , 1 ], +[ 0.019550 , 40 , 3 , 1 ], +[ 0.019795 , 41 , 17 , 1 ], +[ 0.021137 , 1 , 0 , 2 ], +[ 0.021294 , 2 , 1 , 2 ], +[ 0.021534 , 3 , 2 , 2 ], +[ 0.021764 , 4 , 3 , 2 ], +[ 0.022007 , 5 , 4 , 2 ], +[ 0.022251 , 6 , 5 , 2 ], +[ 0.022505 , 7 , 6 , 2 ], +[ 0.022768 , 8 , 7 , 2 ], +[ 0.022989 , 9 , 8 , 2 ], +[ 0.023245 , 10 , 9 , 2 ], +[ 0.023490 , 11 , 10 , 2 ], +[ 0.023747 , 12 , 11 , 2 ], +[ 0.023963 , 13 , 12 , 2 ], +[ 0.024212 , 14 , 13 , 2 ], +[ 0.024972 , 17 , 16 , 2 ], +[ 0.025262 , 18 , 17 , 2 ], +[ 0.025545 , 19 , 0 , 2 ], +[ 0.025701 , 20 , 1 , 2 ], +[ 0.025926 , 21 , 3 , 2 ], +[ 0.026169 , 22 , 4 , 2 ], +[ 0.026486 , 23 , 17 , 2 ], +[ 0.026769 , 24 , 0 , 2 ], +[ 0.026926 , 25 , 1 , 2 ], +[ 0.027151 , 26 , 3 , 2 ], +[ 0.027465 , 27 , 17 , 2 ], +[ 0.027640 , 28 , 3 , 2 ], +[ 0.027955 , 29 , 17 , 2 ], +[ 0.028130 , 30 , 3 , 2 ], +[ 0.028445 , 31 , 17 , 2 ], +[ 0.028728 , 32 , 0 , 2 ], +[ 0.028885 , 33 , 1 , 2 ], +[ 0.029110 , 34 , 3 , 2 ], +[ 0.029424 , 35 , 17 , 2 ], +[ 0.029614 , 36 , 2 , 2 ], +[ 0.029914 , 37 , 17 , 2 ], +[ 0.030087 , 38 , 4 , 2 ], +[ 0.030404 , 39 , 17 , 2 ], +[ 0.030579 , 40 , 3 , 2 ], +[ 0.030894 , 41 , 17 , 2 ], +[ 0.032188 , 1 , 0 , 3 ], +[ 0.032197 , 1 , 0 , 4 ], +[ 0.032341 , 2 , 1 , 3 ], +[ 0.032367 , 2 , 1 , 4 ], +[ 0.032379 , 2 , 1 , 5 ], +[ 0.032576 , 3 , 2 , 3 ], +[ 0.032583 , 3 , 2 , 4 ], +[ 0.032787 , 4 , 3 , 3 ], +[ 0.032796 , 4 , 3 , 4 ], +[ 0.032931 , 16 , 15 , 2 ], +[ 0.033031 , 5 , 4 , 3 ], +[ 0.033052 , 5 , 4 , 4 ], +[ 0.033065 , 5 , 4 , 5 ], +[ 0.033272 , 6 , 5 , 3 ], +[ 0.033460 , 15 , 14 , 3 ], +[ 0.033527 , 7 , 6 , 3 ], +[ 0.033802 , 8 , 7 , 3 ], +[ 0.034029 , 9 , 8 , 3 ], +[ 0.034280 , 10 , 9 , 3 ], +[ 0.034288 , 10 , 9 , 4 ], +[ 0.034525 , 11 , 10 , 3 ], +[ 0.034533 , 11 , 10 , 4 ], +[ 0.034797 , 12 , 11 , 3 ], +[ 0.034989 , 13 , 12 , 3 ], +[ 0.035271 , 14 , 13 , 3 ], +[ 0.036008 , 17 , 16 , 3 ], +[ 0.036442 , 16 , 15 , 3 ], +[ 0.036596 , 19 , 0 , 3 ], +[ 0.036605 , 19 , 0 , 4 ], +[ 0.036749 , 20 , 1 , 3 ], +[ 0.036775 , 20 , 1 , 4 ], +[ 0.036787 , 20 , 1 , 5 ], +[ 0.036949 , 21 , 3 , 3 ], +[ 0.036958 , 21 , 3 , 4 ], +[ 0.037193 , 22 , 4 , 3 ], +[ 0.037215 , 22 , 4 , 4 ], +[ 0.037227 , 22 , 4 , 5 ], +[ 0.037820 , 24 , 0 , 3 ], +[ 0.037829 , 24 , 0 , 4 ], +[ 0.037973 , 25 , 1 , 3 ], +[ 0.037999 , 25 , 1 , 4 ], +[ 0.038011 , 25 , 1 , 5 ], +[ 0.038174 , 26 , 3 , 3 ], +[ 0.038183 , 26 , 3 , 4 ], +[ 0.038663 , 28 , 3 , 3 ], +[ 0.038672 , 28 , 3 , 4 ], +[ 0.039153 , 30 , 3 , 3 ], +[ 0.039162 , 30 , 3 , 4 ], +[ 0.039779 , 32 , 0 , 3 ], +[ 0.039788 , 32 , 0 , 4 ], +[ 0.039932 , 33 , 1 , 3 ], +[ 0.039958 , 33 , 1 , 4 ], +[ 0.039970 , 33 , 1 , 5 ], +[ 0.040133 , 34 , 3 , 3 ], +[ 0.040142 , 34 , 3 , 4 ], +[ 0.040656 , 36 , 2 , 3 ], +[ 0.040663 , 36 , 2 , 4 ], +[ 0.041111 , 38 , 4 , 3 ], +[ 0.041133 , 38 , 4 , 4 ], +[ 0.041145 , 38 , 4 , 5 ], +[ 0.041602 , 40 , 3 , 3 ], +[ 0.041611 , 40 , 3 , 4 ], +[ 0.043401 , 2 , 1 , 6 ], +[ 0.043434 , 1 , 0 , 5 ], +[ 0.043447 , 1 , 0 , 6 ], +[ 0.043815 , 3 , 2 , 5 ], +[ 0.043873 , 4 , 3 , 5 ], +[ 0.043886 , 4 , 3 , 6 ], +[ 0.044082 , 6 , 5 , 4 ], +[ 0.044570 , 7 , 6 , 4 ], +[ 0.044831 , 8 , 7 , 4 ], +[ 0.045090 , 5 , 4 , 6 ], +[ 0.045449 , 10 , 9 , 5 ], +[ 0.045694 , 11 , 10 , 5 ], +[ 0.045839 , 12 , 11 , 4 ], +[ 0.045998 , 9 , 8 , 4 ], +[ 0.046032 , 13 , 12 , 4 ], +[ 0.046300 , 14 , 13 , 4 ], +[ 0.046705 , 16 , 15 , 4 ], +[ 0.047031 , 17 , 16 , 4 ], +[ 0.047809 , 20 , 1 , 6 ], +[ 0.047842 , 19 , 0 , 5 ], +[ 0.047854 , 19 , 0 , 6 ], +[ 0.048035 , 21 , 3 , 5 ], +[ 0.048048 , 21 , 3 , 6 ], +[ 0.049033 , 25 , 1 , 6 ], +[ 0.049066 , 24 , 0 , 5 ], +[ 0.049079 , 24 , 0 , 6 ], +[ 0.049253 , 22 , 4 , 6 ], +[ 0.049260 , 26 , 3 , 5 ], +[ 0.049273 , 26 , 3 , 6 ], +[ 0.049749 , 28 , 3 , 5 ], +[ 0.049763 , 28 , 3 , 6 ], +[ 0.050239 , 30 , 3 , 5 ], +[ 0.050252 , 30 , 3 , 6 ], +[ 0.050992 , 33 , 1 , 6 ], +[ 0.051025 , 32 , 0 , 5 ], +[ 0.051038 , 32 , 0 , 6 ], +[ 0.051219 , 34 , 3 , 5 ], +[ 0.051232 , 34 , 3 , 6 ], +[ 0.051895 , 36 , 2 , 5 ], +[ 0.052688 , 40 , 3 , 5 ], +[ 0.052701 , 40 , 3 , 6 ], +[ 0.053171 , 38 , 4 , 6 ], +[ 0.054470 , 2 , 1 , 7 ], +[ 0.054487 , 2 , 1 , 8 ], +[ 0.054500 , 2 , 1 , 9 ], +[ 0.054521 , 1 , 0 , 7 ], +[ 0.054932 , 4 , 3 , 7 ], +[ 0.055104 , 6 , 5 , 5 ], +[ 0.055245 , 3 , 2 , 6 ], +[ 0.055432 , 8 , 7 , 5 ], +[ 0.055610 , 7 , 6 , 5 ], +[ 0.056678 , 10 , 9 , 6 ], +[ 0.056889 , 12 , 11 , 5 ], +[ 0.056923 , 11 , 10 , 6 ], +[ 0.057032 , 9 , 8 , 5 ], +[ 0.057064 , 13 , 12 , 5 ], +[ 0.057126 , 5 , 4 , 7 ], +[ 0.057323 , 14 , 13 , 5 ], +[ 0.058054 , 17 , 16 , 5 ], +[ 0.058877 , 20 , 1 , 7 ], +[ 0.058895 , 20 , 1 , 8 ], +[ 0.058907 , 20 , 1 , 9 ], +[ 0.058928 , 19 , 0 , 7 ], +[ 0.059094 , 21 , 3 , 7 ], +[ 0.060102 , 25 , 1 , 7 ], +[ 0.060119 , 25 , 1 , 8 ], +[ 0.060132 , 25 , 1 , 9 ], +[ 0.060153 , 24 , 0 , 7 ], +[ 0.060319 , 26 , 3 , 7 ], +[ 0.060372 , 16 , 15 , 5 ], +[ 0.060808 , 28 , 3 , 7 ], +[ 0.061288 , 22 , 4 , 7 ], +[ 0.061298 , 30 , 3 , 7 ], +[ 0.062061 , 33 , 1 , 7 ], +[ 0.062078 , 33 , 1 , 8 ], +[ 0.062091 , 33 , 1 , 9 ], +[ 0.062112 , 32 , 0 , 7 ], +[ 0.062278 , 34 , 3 , 7 ], +[ 0.063325 , 36 , 2 , 6 ], +[ 0.063747 , 40 , 3 , 7 ], +[ 0.065206 , 38 , 4 , 7 ], +[ 0.065542 , 2 , 1 , 10 ], +[ 0.065601 , 1 , 0 , 8 ], +[ 0.065614 , 1 , 0 , 9 ], +[ 0.065626 , 1 , 0 , 10 ], +[ 0.066011 , 4 , 3 , 8 ], +[ 0.066023 , 4 , 3 , 9 ], +[ 0.066036 , 4 , 3 , 10 ], +[ 0.066140 , 6 , 5 , 6 ], +[ 0.066484 , 8 , 7 , 6 ], +[ 0.066651 , 7 , 6 , 6 ], +[ 0.066656 , 3 , 2 , 7 ], +[ 0.067815 , 10 , 9 , 7 ], +[ 0.067928 , 12 , 11 , 6 ], +[ 0.068060 , 11 , 10 , 7 ], +[ 0.068094 , 13 , 12 , 6 ], +[ 0.068149 , 14 , 13 , 6 ], +[ 0.068160 , 5 , 4 , 8 ], +[ 0.068463 , 15 , 14 , 4 ], +[ 0.069950 , 20 , 1 , 10 ], +[ 0.070008 , 19 , 0 , 8 ], +[ 0.070022 , 19 , 0 , 9 ], +[ 0.070034 , 19 , 0 , 10 ], +[ 0.070173 , 21 , 3 , 8 ], +[ 0.070185 , 21 , 3 , 9 ], +[ 0.070199 , 21 , 3 , 10 ], +[ 0.071174 , 25 , 1 , 10 ], +[ 0.071233 , 24 , 0 , 8 ], +[ 0.071246 , 24 , 0 , 9 ], +[ 0.071258 , 24 , 0 , 10 ], +[ 0.071398 , 26 , 3 , 8 ], +[ 0.071410 , 26 , 3 , 9 ], +[ 0.071423 , 26 , 3 , 10 ], +[ 0.071888 , 28 , 3 , 8 ], +[ 0.071899 , 28 , 3 , 9 ], +[ 0.071913 , 28 , 3 , 10 ], +[ 0.072091 , 17 , 16 , 6 ], +[ 0.072322 , 22 , 4 , 8 ], +[ 0.072377 , 30 , 3 , 8 ], +[ 0.072389 , 30 , 3 , 9 ], +[ 0.072402 , 30 , 3 , 10 ], +[ 0.073133 , 33 , 1 , 10 ], +[ 0.073192 , 32 , 0 , 8 ], +[ 0.073205 , 32 , 0 , 9 ], +[ 0.073217 , 32 , 0 , 10 ], +[ 0.073357 , 34 , 3 , 8 ], +[ 0.073369 , 34 , 3 , 9 ], +[ 0.073382 , 34 , 3 , 10 ], +[ 0.074736 , 36 , 2 , 7 ], +[ 0.074826 , 40 , 3 , 8 ], +[ 0.074838 , 40 , 3 , 9 ], +[ 0.074851 , 40 , 3 , 10 ], +[ 0.076240 , 38 , 4 , 8 ], +[ 0.076607 , 2 , 1 , 11 ], +[ 0.076621 , 2 , 1 , 12 ], +[ 0.076633 , 2 , 1 , 13 ], +[ 0.076661 , 1 , 0 , 11 ], +[ 0.077065 , 4 , 3 , 11 ], +[ 0.077162 , 6 , 5 , 7 ], +[ 0.077509 , 8 , 7 , 7 ], +[ 0.077678 , 7 , 6 , 7 ], +[ 0.077745 , 3 , 2 , 8 ], +[ 0.078888 , 10 , 9 , 8 ], +[ 0.078959 , 12 , 11 , 7 ], +[ 0.079126 , 13 , 12 , 7 ], +[ 0.079133 , 11 , 10 , 8 ], +[ 0.079189 , 14 , 13 , 7 ], +[ 0.079212 , 5 , 4 , 9 ], +[ 0.081015 , 20 , 1 , 11 ], +[ 0.081028 , 20 , 1 , 12 ], +[ 0.081041 , 20 , 1 , 13 ], +[ 0.081068 , 19 , 0 , 11 ], +[ 0.081227 , 21 , 3 , 11 ], +[ 0.082239 , 25 , 1 , 11 ], +[ 0.082253 , 25 , 1 , 12 ], +[ 0.082265 , 25 , 1 , 13 ], +[ 0.082293 , 24 , 0 , 11 ], +[ 0.082452 , 26 , 3 , 11 ], +[ 0.082941 , 28 , 3 , 11 ], +[ 0.083374 , 22 , 4 , 9 ], +[ 0.083431 , 30 , 3 , 11 ], +[ 0.084198 , 33 , 1 , 11 ], +[ 0.084212 , 33 , 1 , 12 ], +[ 0.084224 , 33 , 1 , 13 ], +[ 0.084252 , 32 , 0 , 11 ], +[ 0.084411 , 34 , 3 , 11 ], +[ 0.084412 , 15 , 14 , 5 ], +[ 0.085546 , 15 , 14 , 6 ], +[ 0.085825 , 36 , 2 , 8 ], +[ 0.085880 , 40 , 3 , 11 ], +[ 0.086057 , 9 , 8 , 6 ], +[ 0.086065 , 9 , 8 , 7 ], +[ 0.086070 , 9 , 8 , 8 ], +[ 0.086120 , 17 , 16 , 7 ], +[ 0.086549 , 15 , 14 , 7 ], +[ 0.086558 , 15 , 14 , 8 ], +[ 0.087122 , 17 , 16 , 8 ], +[ 0.087127 , 17 , 16 , 9 ], +[ 0.087133 , 17 , 16 , 10 ], +[ 0.087138 , 17 , 16 , 11 ], +[ 0.087148 , 17 , 16 , 12 ], +[ 0.087292 , 38 , 4 , 9 ], +[ 0.087381 , 2 , 1 , 14 ], +[ 0.087410 , 2 , 1 , 15 ], +[ 0.087423 , 2 , 1 , 16 ], +[ 0.087727 , 1 , 0 , 12 ], +[ 0.087740 , 1 , 0 , 13 ], +[ 0.087753 , 1 , 0 , 14 ], +[ 0.088122 , 17 , 16 , 13 ], +[ 0.088152 , 4 , 3 , 12 ], +[ 0.088164 , 4 , 3 , 13 ], +[ 0.088176 , 4 , 3 , 14 ], +[ 0.088200 , 6 , 5 , 8 ], +[ 0.088521 , 8 , 7 , 8 ], +[ 0.088709 , 7 , 6 , 8 ], +[ 0.088855 , 3 , 2 , 9 ], +[ 0.088868 , 3 , 2 , 10 ], +[ 0.088880 , 3 , 2 , 11 ], +[ 0.088893 , 3 , 2 , 12 ], +[ 0.089129 , 17 , 16 , 14 ], +[ 0.089137 , 17 , 16 , 15 ], +[ 0.089142 , 17 , 16 , 16 ], +[ 0.089147 , 17 , 16 , 17 ], +[ 0.089152 , 17 , 16 , 18 ], +[ 0.089550 , 15 , 14 , 9 ], +[ 0.089559 , 15 , 14 , 10 ], +[ 0.089954 , 10 , 9 , 9 ], +[ 0.089997 , 12 , 11 , 8 ], +[ 0.090130 , 17 , 16 , 19 ], +[ 0.090135 , 17 , 16 , 20 ], +[ 0.090141 , 17 , 16 , 21 ], +[ 0.090169 , 13 , 12 , 8 ], +[ 0.090199 , 11 , 10 , 9 ], +[ 0.090232 , 14 , 13 , 8 ], +[ 0.091788 , 20 , 1 , 14 ], +[ 0.091818 , 20 , 1 , 15 ], +[ 0.091830 , 20 , 1 , 16 ], +[ 0.092134 , 19 , 0 , 12 ], +[ 0.092147 , 17 , 16 , 22 ], +[ 0.092148 , 19 , 0 , 13 ], +[ 0.092152 , 17 , 16 , 23 ], +[ 0.092160 , 19 , 0 , 14 ], +[ 0.092186 , 17 , 16 , 24 ], +[ 0.092192 , 17 , 16 , 25 ], +[ 0.092198 , 17 , 16 , 26 ], +[ 0.092272 , 5 , 4 , 10 ], +[ 0.092314 , 21 , 3 , 12 ], +[ 0.092326 , 21 , 3 , 13 ], +[ 0.092338 , 21 , 3 , 14 ], +[ 0.093013 , 25 , 1 , 14 ], +[ 0.093042 , 25 , 1 , 15 ], +[ 0.093055 , 25 , 1 , 16 ], +[ 0.093154 , 17 , 16 , 27 ], +[ 0.093160 , 17 , 16 , 28 ], +[ 0.093359 , 24 , 0 , 12 ], +[ 0.093372 , 24 , 0 , 13 ], +[ 0.093385 , 24 , 0 , 14 ], +[ 0.093539 , 26 , 3 , 12 ], +[ 0.093551 , 26 , 3 , 13 ], +[ 0.093563 , 26 , 3 , 14 ], +[ 0.094029 , 28 , 3 , 12 ], +[ 0.094040 , 28 , 3 , 13 ], +[ 0.094052 , 28 , 3 , 14 ], +[ 0.094518 , 30 , 3 , 12 ], +[ 0.094530 , 30 , 3 , 13 ], +[ 0.094542 , 30 , 3 , 14 ], +[ 0.094972 , 33 , 1 , 14 ], +[ 0.095001 , 33 , 1 , 15 ], +[ 0.095014 , 33 , 1 , 16 ], +[ 0.095318 , 32 , 0 , 12 ], +[ 0.095331 , 32 , 0 , 13 ], +[ 0.095344 , 32 , 0 , 14 ], +[ 0.095498 , 34 , 3 , 12 ], +[ 0.095510 , 34 , 3 , 13 ], +[ 0.095522 , 34 , 3 , 14 ], +[ 0.096434 , 22 , 4 , 10 ], +[ 0.096935 , 36 , 2 , 9 ], +[ 0.096948 , 36 , 2 , 10 ], +[ 0.096960 , 36 , 2 , 11 ], +[ 0.096967 , 40 , 3 , 12 ], +[ 0.096973 , 36 , 2 , 12 ], +[ 0.096979 , 40 , 3 , 13 ], +[ 0.096991 , 40 , 3 , 14 ], +[ 0.098100 , 9 , 8 , 9 ], +[ 0.098440 , 2 , 1 , 17 ], +[ 0.098777 , 1 , 0 , 15 ], +[ 0.098783 , 1 , 0 , 16 ], +[ 0.099207 , 4 , 3 , 15 ], +[ 0.099216 , 4 , 3 , 16 ], +[ 0.099234 , 6 , 5 , 9 ], +[ 0.099540 , 8 , 7 , 9 ], +[ 0.099733 , 7 , 6 , 9 ], +[ 0.099913 , 3 , 2 , 13 ], +[ 0.099918 , 3 , 2 , 14 ], +[ 0.100352 , 38 , 4 , 10 ], +[ 0.101031 , 10 , 9 , 10 ], +[ 0.101033 , 12 , 11 , 9 ], +[ 0.101218 , 13 , 12 , 9 ], +[ 0.101264 , 14 , 13 , 9 ], +[ 0.101276 , 11 , 10 , 10 ], +[ 0.102848 , 20 , 1 , 17 ], +[ 0.103184 , 19 , 0 , 15 ], +[ 0.103190 , 19 , 0 , 16 ], +[ 0.103369 , 21 , 3 , 15 ], +[ 0.103379 , 21 , 3 , 16 ], +[ 0.104072 , 25 , 1 , 17 ], +[ 0.104409 , 24 , 0 , 15 ], +[ 0.104415 , 24 , 0 , 16 ], +[ 0.104594 , 26 , 3 , 15 ], +[ 0.104603 , 26 , 3 , 16 ], +[ 0.105084 , 28 , 3 , 15 ], +[ 0.105093 , 28 , 3 , 16 ], +[ 0.105573 , 30 , 3 , 15 ], +[ 0.105582 , 30 , 3 , 16 ], +[ 0.106031 , 33 , 1 , 17 ], +[ 0.106368 , 32 , 0 , 15 ], +[ 0.106374 , 32 , 0 , 16 ], +[ 0.106553 , 34 , 3 , 15 ], +[ 0.106562 , 34 , 3 , 16 ], +[ 0.107993 , 36 , 2 , 13 ], +[ 0.107998 , 36 , 2 , 14 ], +[ 0.108022 , 40 , 3 , 15 ], +[ 0.108031 , 40 , 3 , 16 ], +[ 0.109544 , 2 , 1 , 18 ], +[ 0.109578 , 2 , 1 , 19 ], +[ 0.109590 , 2 , 1 , 20 ], +[ 0.109842 , 1 , 0 , 17 ], +[ 0.109854 , 1 , 0 , 18 ], +[ 0.109867 , 1 , 0 , 19 ], +[ 0.109879 , 1 , 0 , 20 ], +[ 0.109891 , 1 , 0 , 21 ], +[ 0.109904 , 1 , 0 , 22 ], +[ 0.110264 , 6 , 5 , 10 ], +[ 0.110291 , 4 , 3 , 17 ], +[ 0.110304 , 4 , 3 , 18 ], +[ 0.110316 , 4 , 3 , 19 ], +[ 0.110329 , 4 , 3 , 20 ], +[ 0.110341 , 4 , 3 , 21 ], +[ 0.110353 , 4 , 3 , 22 ], +[ 0.110567 , 8 , 7 , 10 ], +[ 0.110762 , 7 , 6 , 10 ], +[ 0.110992 , 3 , 2 , 15 ], +[ 0.111005 , 3 , 2 , 16 ], +[ 0.111017 , 3 , 2 , 17 ], +[ 0.111029 , 3 , 2 , 18 ], +[ 0.111042 , 3 , 2 , 19 ], +[ 0.111054 , 3 , 2 , 20 ], +[ 0.112076 , 12 , 11 , 10 ], +[ 0.112154 , 10 , 9 , 11 ], +[ 0.112274 , 13 , 12 , 10 ], +[ 0.112291 , 14 , 13 , 10 ], +[ 0.112399 , 11 , 10 , 11 ], +[ 0.113951 , 20 , 1 , 18 ], +[ 0.113986 , 20 , 1 , 19 ], +[ 0.113997 , 20 , 1 , 20 ], +[ 0.114249 , 19 , 0 , 17 ], +[ 0.114262 , 19 , 0 , 18 ], +[ 0.114274 , 19 , 0 , 19 ], +[ 0.114286 , 19 , 0 , 20 ], +[ 0.114299 , 19 , 0 , 21 ], +[ 0.114311 , 19 , 0 , 22 ], +[ 0.114453 , 21 , 3 , 17 ], +[ 0.114466 , 21 , 3 , 18 ], +[ 0.114478 , 21 , 3 , 19 ], +[ 0.114491 , 21 , 3 , 20 ], +[ 0.114503 , 21 , 3 , 21 ], +[ 0.114515 , 21 , 3 , 22 ], +[ 0.115148 , 9 , 8 , 10 ], +[ 0.115176 , 25 , 1 , 18 ], +[ 0.115210 , 25 , 1 , 19 ], +[ 0.115222 , 25 , 1 , 20 ], +[ 0.115474 , 24 , 0 , 17 ], +[ 0.115486 , 24 , 0 , 18 ], +[ 0.115499 , 24 , 0 , 19 ], +[ 0.115511 , 24 , 0 , 20 ], +[ 0.115523 , 24 , 0 , 21 ], +[ 0.115536 , 24 , 0 , 22 ], +[ 0.115678 , 26 , 3 , 17 ], +[ 0.115691 , 26 , 3 , 18 ], +[ 0.115703 , 26 , 3 , 19 ], +[ 0.115716 , 26 , 3 , 20 ], +[ 0.115728 , 26 , 3 , 21 ], +[ 0.115740 , 26 , 3 , 22 ], +[ 0.116167 , 28 , 3 , 17 ], +[ 0.116180 , 28 , 3 , 18 ], +[ 0.116192 , 28 , 3 , 19 ], +[ 0.116206 , 28 , 3 , 20 ], +[ 0.116217 , 28 , 3 , 21 ], +[ 0.116229 , 28 , 3 , 22 ], +[ 0.116657 , 30 , 3 , 17 ], +[ 0.116670 , 30 , 3 , 18 ], +[ 0.116682 , 30 , 3 , 19 ], +[ 0.116695 , 30 , 3 , 20 ], +[ 0.116707 , 30 , 3 , 21 ], +[ 0.116719 , 30 , 3 , 22 ], +[ 0.117135 , 33 , 1 , 18 ], +[ 0.117169 , 33 , 1 , 19 ], +[ 0.117181 , 33 , 1 , 20 ], +[ 0.117433 , 32 , 0 , 17 ], +[ 0.117445 , 32 , 0 , 18 ], +[ 0.117458 , 32 , 0 , 19 ], +[ 0.117470 , 32 , 0 , 20 ], +[ 0.117482 , 32 , 0 , 21 ], +[ 0.117495 , 32 , 0 , 22 ], +[ 0.117637 , 34 , 3 , 17 ], +[ 0.117650 , 34 , 3 , 18 ], +[ 0.117662 , 34 , 3 , 19 ], +[ 0.117675 , 34 , 3 , 20 ], +[ 0.117687 , 34 , 3 , 21 ], +[ 0.117699 , 34 , 3 , 22 ], +[ 0.119072 , 36 , 2 , 15 ], +[ 0.119085 , 36 , 2 , 16 ], +[ 0.119097 , 36 , 2 , 17 ], +[ 0.119106 , 40 , 3 , 17 ], +[ 0.119109 , 36 , 2 , 18 ], +[ 0.119119 , 40 , 3 , 18 ], +[ 0.119122 , 36 , 2 , 19 ], +[ 0.119131 , 40 , 3 , 19 ], +[ 0.119134 , 36 , 2 , 20 ], +[ 0.119144 , 40 , 3 , 20 ], +[ 0.119156 , 40 , 3 , 21 ], +[ 0.119168 , 40 , 3 , 22 ], +[ 0.120605 , 2 , 1 , 21 ], +[ 0.120900 , 1 , 0 , 23 ], +[ 0.120910 , 1 , 0 , 24 ], +[ 0.120914 , 1 , 0 , 25 ], +[ 0.121289 , 6 , 5 , 11 ], +[ 0.121346 , 4 , 3 , 23 ], +[ 0.121352 , 4 , 3 , 24 ], +[ 0.121357 , 4 , 3 , 25 ], +[ 0.121601 , 8 , 7 , 11 ], +[ 0.121787 , 7 , 6 , 11 ], +[ 0.122057 , 3 , 2 , 21 ], +[ 0.122062 , 3 , 2 , 22 ], +[ 0.122066 , 3 , 2 , 23 ], +[ 0.123109 , 12 , 11 , 11 ], +[ 0.123312 , 14 , 13 , 11 ], +[ 0.123322 , 13 , 12 , 11 ], +[ 0.125013 , 20 , 1 , 21 ], +[ 0.125308 , 19 , 0 , 23 ], +[ 0.125318 , 19 , 0 , 24 ], +[ 0.125322 , 19 , 0 , 25 ], +[ 0.125508 , 21 , 3 , 23 ], +[ 0.125514 , 21 , 3 , 24 ], +[ 0.125519 , 21 , 3 , 25 ], +[ 0.125629 , 15 , 14 , 11 ], +[ 0.126237 , 25 , 1 , 21 ], +[ 0.126532 , 24 , 0 , 23 ], +[ 0.126542 , 24 , 0 , 24 ], +[ 0.126546 , 24 , 0 , 25 ], +[ 0.126724 , 9 , 8 , 11 ], +[ 0.126733 , 26 , 3 , 23 ], +[ 0.126739 , 26 , 3 , 24 ], +[ 0.126744 , 26 , 3 , 25 ], +[ 0.127222 , 28 , 3 , 23 ], +[ 0.127229 , 28 , 3 , 24 ], +[ 0.127234 , 28 , 3 , 25 ], +[ 0.127712 , 30 , 3 , 23 ], +[ 0.127718 , 30 , 3 , 24 ], +[ 0.127723 , 30 , 3 , 25 ], +[ 0.128196 , 33 , 1 , 21 ], +[ 0.128491 , 32 , 0 , 23 ], +[ 0.128501 , 32 , 0 , 24 ], +[ 0.128505 , 32 , 0 , 25 ], +[ 0.128692 , 34 , 3 , 23 ], +[ 0.128698 , 34 , 3 , 24 ], +[ 0.128703 , 34 , 3 , 25 ], +[ 0.130137 , 36 , 2 , 21 ], +[ 0.130142 , 36 , 2 , 22 ], +[ 0.130146 , 36 , 2 , 23 ], +[ 0.130161 , 40 , 3 , 23 ], +[ 0.130167 , 40 , 3 , 24 ], +[ 0.130172 , 40 , 3 , 25 ], +[ 0.131565 , 1 , 0 , 26 ], +[ 0.131578 , 1 , 0 , 27 ], +[ 0.131590 , 1 , 0 , 28 ], +[ 0.131603 , 1 , 0 , 29 ], +[ 0.131615 , 1 , 0 , 30 ], +[ 0.131627 , 1 , 0 , 31 ], +[ 0.131640 , 1 , 0 , 32 ], +[ 0.131652 , 1 , 0 , 33 ], +[ 0.131664 , 1 , 0 , 34 ], +[ 0.131689 , 2 , 1 , 22 ], +[ 0.131702 , 2 , 1 , 23 ], +[ 0.131715 , 2 , 1 , 24 ], +[ 0.132022 , 4 , 3 , 26 ], +[ 0.132035 , 4 , 3 , 27 ], +[ 0.132047 , 4 , 3 , 28 ], +[ 0.132060 , 4 , 3 , 29 ], +[ 0.132072 , 4 , 3 , 30 ], +[ 0.132084 , 4 , 3 , 31 ], +[ 0.132097 , 4 , 3 , 32 ], +[ 0.132109 , 4 , 3 , 33 ], +[ 0.132387 , 6 , 5 , 12 ], +[ 0.132399 , 6 , 5 , 13 ], +[ 0.132412 , 6 , 5 , 14 ], +[ 0.132701 , 8 , 7 , 12 ], +[ 0.132714 , 8 , 7 , 13 ], +[ 0.132723 , 3 , 2 , 24 ], +[ 0.132727 , 8 , 7 , 14 ], +[ 0.132735 , 3 , 2 , 25 ], +[ 0.132739 , 8 , 7 , 15 ], +[ 0.132747 , 3 , 2 , 26 ], +[ 0.132751 , 8 , 7 , 16 ], +[ 0.132760 , 3 , 2 , 27 ], +[ 0.132764 , 8 , 7 , 17 ], +[ 0.132772 , 3 , 2 , 28 ], +[ 0.132784 , 3 , 2 , 29 ], +[ 0.132796 , 3 , 2 , 30 ], +[ 0.132809 , 3 , 2 , 31 ], +[ 0.132821 , 3 , 2 , 32 ], +[ 0.132907 , 7 , 6 , 12 ], +[ 0.132920 , 7 , 6 , 13 ], +[ 0.132932 , 7 , 6 , 14 ], +[ 0.132944 , 7 , 6 , 15 ], +[ 0.132957 , 7 , 6 , 16 ], +[ 0.132969 , 7 , 6 , 17 ], +[ 0.134138 , 12 , 11 , 12 ], +[ 0.134162 , 14 , 13 , 12 ], +[ 0.134377 , 13 , 12 , 12 ], +[ 0.135972 , 19 , 0 , 26 ], +[ 0.135986 , 19 , 0 , 27 ], +[ 0.135997 , 19 , 0 , 28 ], +[ 0.136011 , 19 , 0 , 29 ], +[ 0.136023 , 19 , 0 , 30 ], +[ 0.136034 , 19 , 0 , 31 ], +[ 0.136048 , 19 , 0 , 32 ], +[ 0.136059 , 19 , 0 , 33 ], +[ 0.136071 , 19 , 0 , 34 ], +[ 0.136096 , 20 , 1 , 22 ], +[ 0.136110 , 20 , 1 , 23 ], +[ 0.136122 , 20 , 1 , 24 ], +[ 0.136184 , 21 , 3 , 26 ], +[ 0.136197 , 21 , 3 , 27 ], +[ 0.136209 , 21 , 3 , 28 ], +[ 0.136222 , 21 , 3 , 29 ], +[ 0.136235 , 21 , 3 , 30 ], +[ 0.136246 , 21 , 3 , 31 ], +[ 0.136259 , 21 , 3 , 32 ], +[ 0.136271 , 21 , 3 , 33 ], +[ 0.137197 , 24 , 0 , 26 ], +[ 0.137210 , 24 , 0 , 27 ], +[ 0.137222 , 24 , 0 , 28 ], +[ 0.137235 , 24 , 0 , 29 ], +[ 0.137247 , 24 , 0 , 30 ], +[ 0.137259 , 24 , 0 , 31 ], +[ 0.137272 , 24 , 0 , 32 ], +[ 0.137284 , 24 , 0 , 33 ], +[ 0.137296 , 24 , 0 , 34 ], +[ 0.137321 , 25 , 1 , 22 ], +[ 0.137334 , 25 , 1 , 23 ], +[ 0.137347 , 25 , 1 , 24 ], +[ 0.137409 , 26 , 3 , 26 ], +[ 0.137422 , 26 , 3 , 27 ], +[ 0.137434 , 26 , 3 , 28 ], +[ 0.137447 , 26 , 3 , 29 ], +[ 0.137459 , 26 , 3 , 30 ], +[ 0.137471 , 26 , 3 , 31 ], +[ 0.137484 , 26 , 3 , 32 ], +[ 0.137496 , 26 , 3 , 33 ], +[ 0.137898 , 28 , 3 , 26 ], +[ 0.137911 , 28 , 3 , 27 ], +[ 0.137924 , 28 , 3 , 28 ], +[ 0.137936 , 28 , 3 , 29 ], +[ 0.137949 , 28 , 3 , 30 ], +[ 0.137960 , 28 , 3 , 31 ], +[ 0.137973 , 28 , 3 , 32 ], +[ 0.137986 , 28 , 3 , 33 ], +[ 0.138388 , 30 , 3 , 26 ], +[ 0.138401 , 30 , 3 , 27 ], +[ 0.138413 , 30 , 3 , 28 ], +[ 0.138426 , 30 , 3 , 29 ], +[ 0.138438 , 30 , 3 , 30 ], +[ 0.138450 , 30 , 3 , 31 ], +[ 0.138463 , 30 , 3 , 32 ], +[ 0.138475 , 30 , 3 , 33 ], +[ 0.138784 , 9 , 8 , 12 ], +[ 0.139156 , 32 , 0 , 26 ], +[ 0.139169 , 32 , 0 , 27 ], +[ 0.139181 , 32 , 0 , 28 ], +[ 0.139194 , 32 , 0 , 29 ], +[ 0.139206 , 32 , 0 , 30 ], +[ 0.139218 , 32 , 0 , 31 ], +[ 0.139231 , 32 , 0 , 32 ], +[ 0.139243 , 32 , 0 , 33 ], +[ 0.139255 , 32 , 0 , 34 ], +[ 0.139280 , 33 , 1 , 22 ], +[ 0.139293 , 33 , 1 , 23 ], +[ 0.139306 , 33 , 1 , 24 ], +[ 0.139368 , 34 , 3 , 26 ], +[ 0.139381 , 34 , 3 , 27 ], +[ 0.139393 , 34 , 3 , 28 ], +[ 0.139406 , 34 , 3 , 29 ], +[ 0.139418 , 34 , 3 , 30 ], +[ 0.139430 , 34 , 3 , 31 ], +[ 0.139443 , 34 , 3 , 32 ], +[ 0.139455 , 34 , 3 , 33 ], +[ 0.140803 , 36 , 2 , 24 ], +[ 0.140815 , 36 , 2 , 25 ], +[ 0.140827 , 36 , 2 , 26 ], +[ 0.140837 , 40 , 3 , 26 ], +[ 0.140840 , 36 , 2 , 27 ], +[ 0.140850 , 40 , 3 , 27 ], +[ 0.140852 , 36 , 2 , 28 ], +[ 0.140862 , 40 , 3 , 28 ], +[ 0.140864 , 36 , 2 , 29 ], +[ 0.140875 , 40 , 3 , 29 ], +[ 0.140876 , 36 , 2 , 30 ], +[ 0.140887 , 40 , 3 , 30 ], +[ 0.140889 , 36 , 2 , 31 ], +[ 0.140899 , 40 , 3 , 31 ], +[ 0.140901 , 36 , 2 , 32 ], +[ 0.140912 , 40 , 3 , 32 ], +[ 0.140924 , 40 , 3 , 33 ], +[ 0.142209 , 1 , 0 , 35 ], +[ 0.142215 , 1 , 0 , 36 ], +[ 0.142220 , 1 , 0 , 37 ], +[ 0.142225 , 1 , 0 , 38 ], +[ 0.142666 , 4 , 3 , 34 ], +[ 0.142671 , 4 , 3 , 35 ], +[ 0.142676 , 4 , 3 , 36 ], +[ 0.142681 , 4 , 3 , 37 ], +[ 0.142740 , 2 , 1 , 25 ], +[ 0.142745 , 2 , 1 , 26 ], +[ 0.143371 , 3 , 2 , 33 ], +[ 0.143376 , 3 , 2 , 34 ], +[ 0.143382 , 3 , 2 , 35 ], +[ 0.143387 , 3 , 2 , 36 ], +[ 0.143444 , 6 , 5 , 15 ], +[ 0.143449 , 6 , 5 , 16 ], +[ 0.143757 , 8 , 7 , 18 ], +[ 0.143762 , 8 , 7 , 19 ], +[ 0.143767 , 8 , 7 , 20 ], +[ 0.143968 , 7 , 6 , 18 ], +[ 0.143974 , 7 , 6 , 19 ], +[ 0.143978 , 7 , 6 , 20 ], +[ 0.145210 , 12 , 11 , 13 ], +[ 0.145223 , 14 , 13 , 13 ], +[ 0.145224 , 12 , 11 , 14 ], +[ 0.145236 , 12 , 11 , 15 ], +[ 0.145237 , 14 , 13 , 14 ], +[ 0.145249 , 14 , 13 , 15 ], +[ 0.145262 , 14 , 13 , 16 ], +[ 0.145275 , 14 , 13 , 17 ], +[ 0.145287 , 14 , 13 , 18 ], +[ 0.145449 , 13 , 12 , 13 ], +[ 0.145462 , 13 , 12 , 14 ], +[ 0.145475 , 13 , 12 , 15 ], +[ 0.145488 , 13 , 12 , 16 ], +[ 0.145501 , 13 , 12 , 17 ], +[ 0.145514 , 13 , 12 , 18 ], +[ 0.146616 , 19 , 0 , 35 ], +[ 0.146622 , 19 , 0 , 36 ], +[ 0.146627 , 19 , 0 , 37 ], +[ 0.146632 , 19 , 0 , 38 ], +[ 0.146828 , 21 , 3 , 34 ], +[ 0.146833 , 21 , 3 , 35 ], +[ 0.146838 , 21 , 3 , 36 ], +[ 0.146843 , 21 , 3 , 37 ], +[ 0.147147 , 20 , 1 , 25 ], +[ 0.147152 , 20 , 1 , 26 ], +[ 0.147841 , 24 , 0 , 35 ], +[ 0.147847 , 24 , 0 , 36 ], +[ 0.147852 , 24 , 0 , 37 ], +[ 0.147857 , 24 , 0 , 38 ], +[ 0.148053 , 26 , 3 , 34 ], +[ 0.148058 , 26 , 3 , 35 ], +[ 0.148063 , 26 , 3 , 36 ], +[ 0.148068 , 26 , 3 , 37 ], +[ 0.148372 , 25 , 1 , 25 ], +[ 0.148377 , 25 , 1 , 26 ], +[ 0.148542 , 28 , 3 , 34 ], +[ 0.148547 , 28 , 3 , 35 ], +[ 0.148552 , 28 , 3 , 36 ], +[ 0.148557 , 28 , 3 , 37 ], +[ 0.149032 , 30 , 3 , 34 ], +[ 0.149037 , 30 , 3 , 35 ], +[ 0.149042 , 30 , 3 , 36 ], +[ 0.149047 , 30 , 3 , 37 ], +[ 0.149195 , 17 , 16 , 29 ], +[ 0.149800 , 32 , 0 , 35 ], +[ 0.149806 , 32 , 0 , 36 ], +[ 0.149811 , 32 , 0 , 37 ], +[ 0.149816 , 32 , 0 , 38 ], +[ 0.150012 , 34 , 3 , 34 ], +[ 0.150017 , 34 , 3 , 35 ], +[ 0.150022 , 34 , 3 , 36 ], +[ 0.150027 , 34 , 3 , 37 ], +[ 0.150331 , 33 , 1 , 25 ], +[ 0.150336 , 33 , 1 , 26 ], +[ 0.151451 , 36 , 2 , 33 ], +[ 0.151456 , 36 , 2 , 34 ], +[ 0.151462 , 36 , 2 , 35 ], +[ 0.151467 , 36 , 2 , 36 ], +[ 0.151481 , 40 , 3 , 34 ], +[ 0.151486 , 40 , 3 , 35 ], +[ 0.151491 , 40 , 3 , 36 ], +[ 0.151496 , 40 , 3 , 37 ], +[ 0.151863 , 9 , 8 , 13 ], +[ 0.153285 , 1 , 0 , 39 ], +[ 0.153298 , 1 , 0 , 40 ], +[ 0.153299 , 1 , 0 , 41 ], +[ 0.153833 , 2 , 1 , 27 ], +[ 0.153846 , 2 , 1 , 28 ], +[ 0.153858 , 2 , 1 , 29 ], +[ 0.153871 , 2 , 1 , 30 ], +[ 0.153883 , 2 , 1 , 31 ], +[ 0.153895 , 2 , 1 , 32 ], +[ 0.154383 , 8 , 7 , 21 ], +[ 0.154396 , 8 , 7 , 22 ], +[ 0.154408 , 8 , 7 , 23 ], +[ 0.154420 , 8 , 7 , 24 ], +[ 0.154450 , 3 , 2 , 37 ], +[ 0.154462 , 3 , 2 , 38 ], +[ 0.154469 , 6 , 5 , 17 ], +[ 0.154475 , 3 , 2 , 39 ], +[ 0.154475 , 6 , 5 , 18 ], +[ 0.154487 , 3 , 2 , 40 ], +[ 0.154499 , 3 , 2 , 41 ], +[ 0.154512 , 3 , 2 , 42 ], +[ 0.154524 , 3 , 2 , 43 ], +[ 0.154634 , 7 , 6 , 21 ], +[ 0.154647 , 7 , 6 , 22 ], +[ 0.154659 , 7 , 6 , 23 ], +[ 0.154672 , 7 , 6 , 24 ], +[ 0.154684 , 7 , 6 , 25 ], +[ 0.154696 , 7 , 6 , 26 ], +[ 0.154709 , 7 , 6 , 27 ], +[ 0.154721 , 7 , 6 , 28 ], +[ 0.154733 , 7 , 6 , 29 ], +[ 0.156276 , 12 , 11 , 16 ], +[ 0.156283 , 12 , 11 , 17 ], +[ 0.156286 , 14 , 13 , 19 ], +[ 0.156292 , 14 , 13 , 20 ], +[ 0.156297 , 14 , 13 , 21 ], +[ 0.156516 , 13 , 12 , 19 ], +[ 0.156523 , 13 , 12 , 20 ], +[ 0.156529 , 13 , 12 , 21 ], +[ 0.157692 , 19 , 0 , 39 ], +[ 0.157706 , 19 , 0 , 40 ], +[ 0.157706 , 19 , 0 , 41 ], +[ 0.158241 , 20 , 1 , 27 ], +[ 0.158253 , 20 , 1 , 28 ], +[ 0.158266 , 20 , 1 , 29 ], +[ 0.158278 , 20 , 1 , 30 ], +[ 0.158291 , 20 , 1 , 31 ], +[ 0.158303 , 20 , 1 , 32 ], +[ 0.158917 , 24 , 0 , 39 ], +[ 0.158930 , 24 , 0 , 40 ], +[ 0.158931 , 24 , 0 , 41 ], +[ 0.159465 , 25 , 1 , 27 ], +[ 0.159478 , 25 , 1 , 28 ], +[ 0.159490 , 25 , 1 , 29 ], +[ 0.159503 , 25 , 1 , 30 ], +[ 0.159515 , 25 , 1 , 31 ], +[ 0.159527 , 25 , 1 , 32 ], +[ 0.160238 , 17 , 16 , 30 ], +[ 0.160643 , 15 , 14 , 12 ], +[ 0.160876 , 32 , 0 , 39 ], +[ 0.160889 , 32 , 0 , 40 ], +[ 0.160890 , 32 , 0 , 41 ], +[ 0.161424 , 33 , 1 , 27 ], +[ 0.161437 , 33 , 1 , 28 ], +[ 0.161449 , 33 , 1 , 29 ], +[ 0.161462 , 33 , 1 , 30 ], +[ 0.161474 , 33 , 1 , 31 ], +[ 0.161486 , 33 , 1 , 32 ], +[ 0.162530 , 36 , 2 , 37 ], +[ 0.162542 , 36 , 2 , 38 ], +[ 0.162555 , 36 , 2 , 39 ], +[ 0.162567 , 36 , 2 , 40 ], +[ 0.162579 , 36 , 2 , 41 ], +[ 0.162592 , 36 , 2 , 42 ], +[ 0.162604 , 36 , 2 , 43 ], +[ 0.162904 , 9 , 8 , 14 ], +[ 0.164347 , 1 , 0 , 42 ], +[ 0.164353 , 1 , 0 , 43 ], +[ 0.164889 , 2 , 1 , 33 ], +[ 0.164895 , 2 , 1 , 34 ], +[ 0.164901 , 2 , 1 , 35 ], +[ 0.165168 , 3 , 2 , 44 ], +[ 0.165173 , 3 , 2 , 45 ], +[ 0.165178 , 3 , 2 , 46 ], +[ 0.165199 , 3 , 2 , 47 ], +[ 0.165211 , 3 , 2 , 48 ], +[ 0.165224 , 3 , 2 , 49 ], +[ 0.165236 , 3 , 2 , 50 ], +[ 0.165248 , 3 , 2 , 51 ], +[ 0.165280 , 7 , 6 , 30 ], +[ 0.165286 , 7 , 6 , 31 ], +[ 0.165290 , 7 , 6 , 32 ], +[ 0.165295 , 7 , 6 , 33 ], +[ 0.165404 , 8 , 7 , 25 ], +[ 0.165410 , 8 , 7 , 26 ], +[ 0.165492 , 6 , 5 , 19 ], +[ 0.166940 , 14 , 13 , 22 ], +[ 0.166952 , 14 , 13 , 23 ], +[ 0.166965 , 14 , 13 , 24 ], +[ 0.166977 , 14 , 13 , 25 ], +[ 0.166989 , 14 , 13 , 26 ], +[ 0.167002 , 14 , 13 , 27 ], +[ 0.167014 , 14 , 13 , 28 ], +[ 0.167027 , 14 , 13 , 29 ], +[ 0.167039 , 14 , 13 , 30 ], +[ 0.167318 , 12 , 11 , 18 ], +[ 0.167583 , 13 , 12 , 22 ], +[ 0.167596 , 13 , 12 , 23 ], +[ 0.167610 , 13 , 12 , 24 ], +[ 0.167623 , 13 , 12 , 25 ], +[ 0.167635 , 13 , 12 , 26 ], +[ 0.167648 , 13 , 12 , 27 ], +[ 0.168755 , 19 , 0 , 42 ], +[ 0.168760 , 19 , 0 , 43 ], +[ 0.169296 , 20 , 1 , 33 ], +[ 0.169302 , 20 , 1 , 34 ], +[ 0.169308 , 20 , 1 , 35 ], +[ 0.169979 , 24 , 0 , 42 ], +[ 0.169985 , 24 , 0 , 43 ], +[ 0.170521 , 25 , 1 , 33 ], +[ 0.170527 , 25 , 1 , 34 ], +[ 0.170533 , 25 , 1 , 35 ], +[ 0.171938 , 32 , 0 , 42 ], +[ 0.171944 , 32 , 0 , 43 ], +[ 0.172480 , 33 , 1 , 33 ], +[ 0.172486 , 33 , 1 , 34 ], +[ 0.172492 , 33 , 1 , 35 ], +[ 0.173248 , 36 , 2 , 44 ], +[ 0.173253 , 36 , 2 , 45 ], +[ 0.173258 , 36 , 2 , 46 ], +[ 0.173279 , 36 , 2 , 47 ], +[ 0.173291 , 36 , 2 , 48 ], +[ 0.173304 , 36 , 2 , 49 ], +[ 0.173316 , 36 , 2 , 50 ], +[ 0.173328 , 36 , 2 , 51 ], +[ 0.174314 , 17 , 16 , 31 ], +[ 0.175402 , 1 , 0 , 44 ], +[ 0.175545 , 2 , 1 , 36 ], +[ 0.175558 , 2 , 1 , 37 ], +[ 0.175570 , 2 , 1 , 38 ], +[ 0.175582 , 2 , 1 , 39 ], +[ 0.175595 , 2 , 1 , 40 ], +[ 0.175607 , 2 , 1 , 41 ], +[ 0.175619 , 2 , 1 , 42 ], +[ 0.175632 , 2 , 1 , 43 ], +[ 0.175644 , 2 , 1 , 44 ], +[ 0.175823 , 3 , 2 , 52 ], +[ 0.175831 , 3 , 2 , 53 ], +[ 0.175933 , 7 , 6 , 34 ], +[ 0.175945 , 7 , 6 , 35 ], +[ 0.175955 , 9 , 8 , 15 ], +[ 0.175957 , 7 , 6 , 36 ], +[ 0.175970 , 7 , 6 , 37 ], +[ 0.175982 , 7 , 6 , 38 ], +[ 0.175994 , 7 , 6 , 39 ], +[ 0.176007 , 7 , 6 , 40 ], +[ 0.176019 , 7 , 6 , 41 ], +[ 0.176031 , 7 , 6 , 42 ], +[ 0.176431 , 8 , 7 , 27 ], +[ 0.176437 , 8 , 7 , 28 ], +[ 0.176444 , 7 , 6 , 43 ], +[ 0.176457 , 7 , 6 , 44 ], +[ 0.176470 , 7 , 6 , 45 ], +[ 0.176535 , 6 , 5 , 20 ], +[ 0.177594 , 14 , 13 , 31 ], +[ 0.177600 , 14 , 13 , 32 ], +[ 0.177605 , 14 , 13 , 33 ], +[ 0.177610 , 14 , 13 , 34 ], +[ 0.178374 , 12 , 11 , 19 ], +[ 0.178379 , 12 , 11 , 20 ], +[ 0.178653 , 13 , 12 , 28 ], +[ 0.178660 , 13 , 12 , 29 ], +[ 0.178666 , 13 , 12 , 30 ], +[ 0.179810 , 19 , 0 , 44 ], +[ 0.179953 , 20 , 1 , 36 ], +[ 0.179965 , 20 , 1 , 37 ], +[ 0.179977 , 20 , 1 , 38 ], +[ 0.179990 , 20 , 1 , 39 ], +[ 0.180002 , 20 , 1 , 40 ], +[ 0.180015 , 20 , 1 , 41 ], +[ 0.180026 , 20 , 1 , 42 ], +[ 0.180040 , 20 , 1 , 43 ], +[ 0.180052 , 20 , 1 , 44 ], +[ 0.181034 , 24 , 0 , 44 ], +[ 0.181177 , 25 , 1 , 36 ], +[ 0.181190 , 25 , 1 , 37 ], +[ 0.181202 , 25 , 1 , 38 ], +[ 0.181214 , 25 , 1 , 39 ], +[ 0.181227 , 25 , 1 , 40 ], +[ 0.181239 , 25 , 1 , 41 ], +[ 0.181251 , 25 , 1 , 42 ], +[ 0.181264 , 25 , 1 , 43 ], +[ 0.181276 , 25 , 1 , 44 ], +[ 0.182993 , 32 , 0 , 44 ], +[ 0.183136 , 33 , 1 , 36 ], +[ 0.183149 , 33 , 1 , 37 ], +[ 0.183161 , 33 , 1 , 38 ], +[ 0.183173 , 33 , 1 , 39 ], +[ 0.183186 , 33 , 1 , 40 ], +[ 0.183198 , 33 , 1 , 41 ], +[ 0.183210 , 33 , 1 , 42 ], +[ 0.183223 , 33 , 1 , 43 ], +[ 0.183235 , 33 , 1 , 44 ], +[ 0.183903 , 36 , 2 , 52 ], +[ 0.183911 , 36 , 2 , 53 ], +[ 0.185366 , 17 , 16 , 32 ], +[ 0.186097 , 7 , 6 , 46 ], +[ 0.186101 , 7 , 6 , 47 ], +[ 0.186189 , 2 , 1 , 45 ], +[ 0.186194 , 2 , 1 , 46 ], +[ 0.186199 , 2 , 1 , 47 ], +[ 0.186205 , 2 , 1 , 48 ], +[ 0.186590 , 7 , 6 , 48 ], +[ 0.186595 , 7 , 6 , 49 ], +[ 0.186600 , 7 , 6 , 50 ], +[ 0.186604 , 7 , 6 , 51 ], +[ 0.186898 , 3 , 2 , 54 ], +[ 0.186912 , 3 , 2 , 55 ], +[ 0.186924 , 3 , 2 , 56 ], +[ 0.187451 , 8 , 7 , 29 ], +[ 0.188232 , 14 , 13 , 35 ], +[ 0.188244 , 14 , 13 , 36 ], +[ 0.188257 , 14 , 13 , 37 ], +[ 0.188269 , 14 , 13 , 38 ], +[ 0.188281 , 14 , 13 , 39 ], +[ 0.188294 , 14 , 13 , 40 ], +[ 0.188306 , 14 , 13 , 41 ], +[ 0.188318 , 14 , 13 , 42 ], +[ 0.188331 , 14 , 13 , 43 ], +[ 0.188343 , 14 , 13 , 44 ], +[ 0.188355 , 14 , 13 , 45 ], +[ 0.188368 , 14 , 13 , 46 ], +[ 0.188371 , 17 , 16 , 33 ], +[ 0.189410 , 12 , 11 , 21 ], +[ 0.189695 , 13 , 12 , 31 ], +[ 0.190597 , 20 , 1 , 45 ], +[ 0.190602 , 20 , 1 , 46 ], +[ 0.190607 , 20 , 1 , 47 ], +[ 0.190613 , 20 , 1 , 48 ], +[ 0.190655 , 15 , 14 , 13 ], +[ 0.191821 , 25 , 1 , 45 ], +[ 0.191826 , 25 , 1 , 46 ], +[ 0.191831 , 25 , 1 , 47 ], +[ 0.191837 , 25 , 1 , 48 ], +[ 0.193780 , 33 , 1 , 45 ], +[ 0.193785 , 33 , 1 , 46 ], +[ 0.193790 , 33 , 1 , 47 ], +[ 0.193796 , 33 , 1 , 48 ], +[ 0.194978 , 36 , 2 , 54 ], +[ 0.194992 , 36 , 2 , 55 ], +[ 0.195004 , 36 , 2 , 56 ], +[ 0.196755 , 7 , 6 , 52 ], +[ 0.196768 , 7 , 6 , 53 ], +[ 0.196780 , 7 , 6 , 54 ], +[ 0.196796 , 7 , 6 , 55 ], +[ 0.196809 , 7 , 6 , 56 ], +[ 0.196821 , 7 , 6 , 57 ], +[ 0.196834 , 7 , 6 , 58 ], +[ 0.196846 , 7 , 6 , 59 ], +[ 0.196858 , 7 , 6 , 60 ], +[ 0.196871 , 7 , 6 , 61 ], +[ 0.196883 , 7 , 6 , 62 ], +[ 0.196895 , 7 , 6 , 63 ], +[ 0.196908 , 7 , 6 , 64 ], +[ 0.196920 , 7 , 6 , 65 ], +[ 0.196932 , 7 , 6 , 66 ], +[ 0.197192 , 2 , 1 , 49 ], +[ 0.197204 , 2 , 1 , 50 ], +[ 0.197205 , 2 , 1 , 51 ], +[ 0.197349 , 7 , 6 , 67 ], +[ 0.197361 , 7 , 6 , 68 ], +[ 0.197374 , 7 , 6 , 69 ], +[ 0.197601 , 3 , 2 , 57 ], +[ 0.197632 , 3 , 2 , 58 ], +[ 0.197644 , 3 , 2 , 59 ], +[ 0.197657 , 3 , 2 , 60 ], +[ 0.197669 , 3 , 2 , 61 ], +[ 0.197681 , 3 , 2 , 62 ], +[ 0.197694 , 3 , 2 , 63 ], +[ 0.197706 , 3 , 2 , 64 ], +[ 0.197718 , 3 , 2 , 65 ], +[ 0.197731 , 3 , 2 , 66 ], +[ 0.197743 , 3 , 2 , 67 ], +[ 0.197755 , 3 , 2 , 68 ], +[ 0.197768 , 3 , 2 , 69 ], +[ 0.197780 , 3 , 2 , 70 ], +[ 0.197792 , 3 , 2 , 71 ], +[ 0.197804 , 3 , 2 , 72 ], +[ 0.197817 , 3 , 2 , 73 ], +[ 0.198231 , 3 , 2 , 74 ], +[ 0.198480 , 8 , 7 , 30 ], +[ 0.198868 , 14 , 13 , 47 ], +[ 0.198873 , 14 , 13 , 48 ], +[ 0.198878 , 14 , 13 , 49 ], +[ 0.198884 , 14 , 13 , 50 ], +[ 0.198890 , 14 , 13 , 51 ], +[ 0.199381 , 14 , 13 , 52 ], +[ 0.200412 , 17 , 16 , 34 ], +[ 0.200454 , 12 , 11 , 22 ], +[ 0.200722 , 13 , 12 , 32 ], +[ 0.200727 , 13 , 12 , 33 ], +[ 0.201600 , 20 , 1 , 49 ], +[ 0.201611 , 20 , 1 , 50 ], +[ 0.201612 , 20 , 1 , 51 ], +[ 0.202824 , 25 , 1 , 49 ], +[ 0.202836 , 25 , 1 , 50 ], +[ 0.202837 , 25 , 1 , 51 ], +[ 0.204783 , 33 , 1 , 49 ], +[ 0.204795 , 33 , 1 , 50 ], +[ 0.204796 , 33 , 1 , 51 ], +[ 0.205681 , 36 , 2 , 57 ], +[ 0.205712 , 36 , 2 , 58 ], +[ 0.205724 , 36 , 2 , 59 ], +[ 0.205737 , 36 , 2 , 60 ], +[ 0.205749 , 36 , 2 , 61 ], +[ 0.205761 , 36 , 2 , 62 ], +[ 0.205774 , 36 , 2 , 63 ], +[ 0.205786 , 36 , 2 , 64 ], +[ 0.205798 , 36 , 2 , 65 ], +[ 0.205811 , 36 , 2 , 66 ], +[ 0.205823 , 36 , 2 , 67 ], +[ 0.205835 , 36 , 2 , 68 ], +[ 0.205848 , 36 , 2 , 69 ], +[ 0.205860 , 36 , 2 , 70 ], +[ 0.205872 , 36 , 2 , 71 ], +[ 0.205884 , 36 , 2 , 72 ], +[ 0.205897 , 36 , 2 , 73 ], +[ 0.206311 , 36 , 2 , 74 ], +[ 0.207015 , 7 , 6 , 70 ], +[ 0.207019 , 7 , 6 , 71 ], +[ 0.207024 , 7 , 6 , 72 ], +[ 0.207029 , 7 , 6 , 73 ], +[ 0.207034 , 7 , 6 , 74 ], +[ 0.207039 , 7 , 6 , 75 ], +[ 0.207044 , 7 , 6 , 76 ], +[ 0.207529 , 7 , 6 , 77 ], +[ 0.207535 , 7 , 6 , 78 ], +[ 0.207752 , 15 , 14 , 14 ], +[ 0.207762 , 15 , 14 , 15 ], +[ 0.207771 , 15 , 14 , 16 ], +[ 0.207871 , 3 , 2 , 75 ], +[ 0.207881 , 3 , 2 , 76 ], +[ 0.207886 , 3 , 2 , 77 ], +[ 0.207891 , 3 , 2 , 78 ], +[ 0.207896 , 3 , 2 , 79 ], +[ 0.207901 , 3 , 2 , 80 ], +[ 0.208210 , 2 , 1 , 52 ], +[ 0.208220 , 2 , 1 , 53 ], +[ 0.208392 , 3 , 2 , 81 ], +[ 0.208397 , 3 , 2 , 82 ], +[ 0.208749 , 15 , 14 , 17 ], +[ 0.209512 , 14 , 13 , 53 ], +[ 0.209524 , 14 , 13 , 54 ], +[ 0.209537 , 14 , 13 , 55 ], +[ 0.209550 , 14 , 13 , 56 ], +[ 0.209562 , 14 , 13 , 57 ], +[ 0.209575 , 14 , 13 , 58 ], +[ 0.209587 , 14 , 13 , 59 ], +[ 0.209599 , 14 , 13 , 60 ], +[ 0.209612 , 14 , 13 , 61 ], +[ 0.209624 , 14 , 13 , 62 ], +[ 0.209636 , 14 , 13 , 63 ], +[ 0.209648 , 14 , 13 , 64 ], +[ 0.209661 , 14 , 13 , 65 ], +[ 0.209673 , 14 , 13 , 66 ], +[ 0.209685 , 14 , 13 , 67 ], +[ 0.209933 , 14 , 13 , 68 ], +[ 0.210447 , 14 , 13 , 69 ], +[ 0.210460 , 14 , 13 , 70 ], +[ 0.210746 , 15 , 14 , 18 ], +[ 0.211437 , 17 , 16 , 35 ], +[ 0.211755 , 13 , 12 , 34 ], +[ 0.212618 , 20 , 1 , 52 ], +[ 0.212628 , 20 , 1 , 53 ], +[ 0.213842 , 25 , 1 , 52 ], +[ 0.213852 , 25 , 1 , 53 ], +[ 0.215801 , 33 , 1 , 52 ], +[ 0.215811 , 33 , 1 , 53 ], +[ 0.215951 , 36 , 2 , 75 ], +[ 0.215961 , 36 , 2 , 76 ], +[ 0.215966 , 36 , 2 , 77 ], +[ 0.215971 , 36 , 2 , 78 ], +[ 0.215976 , 36 , 2 , 79 ], +[ 0.215981 , 36 , 2 , 80 ], +[ 0.216472 , 36 , 2 , 81 ], +[ 0.216477 , 36 , 2 , 82 ], +[ 0.217229 , 7 , 6 , 79 ], +[ 0.217242 , 7 , 6 , 80 ], +[ 0.217254 , 7 , 6 , 81 ], +[ 0.217266 , 7 , 6 , 82 ], +[ 0.217279 , 7 , 6 , 83 ], +[ 0.217291 , 7 , 6 , 84 ], +[ 0.217303 , 7 , 6 , 85 ], +[ 0.217316 , 7 , 6 , 86 ], +[ 0.217328 , 7 , 6 , 87 ], +[ 0.217340 , 7 , 6 , 88 ], +[ 0.217352 , 7 , 6 , 89 ], +[ 0.217365 , 7 , 6 , 90 ], +[ 0.217377 , 7 , 6 , 91 ], +[ 0.217389 , 7 , 6 , 92 ], +[ 0.217633 , 7 , 6 , 93 ], +[ 0.217646 , 7 , 6 , 94 ], +[ 0.217658 , 7 , 6 , 95 ], +[ 0.217671 , 7 , 6 , 96 ], +[ 0.217683 , 7 , 6 , 97 ], +[ 0.217695 , 7 , 6 , 98 ], +[ 0.217707 , 7 , 6 , 99 ], +[ 0.218529 , 3 , 2 , 83 ], +[ 0.218542 , 3 , 2 , 84 ], +[ 0.218554 , 3 , 2 , 85 ], +[ 0.218566 , 3 , 2 , 86 ], +[ 0.218579 , 3 , 2 , 87 ], +[ 0.218591 , 3 , 2 , 88 ], +[ 0.218603 , 3 , 2 , 89 ], +[ 0.218616 , 3 , 2 , 90 ], +[ 0.218628 , 3 , 2 , 91 ], +[ 0.219235 , 2 , 1 , 54 ], +[ 0.219788 , 14 , 13 , 71 ], +[ 0.219794 , 14 , 13 , 72 ], +[ 0.219800 , 14 , 13 , 73 ], +[ 0.219805 , 14 , 13 , 74 ], +[ 0.219810 , 14 , 13 , 75 ], +[ 0.220024 , 9 , 8 , 16 ], +[ 0.220499 , 14 , 13 , 76 ], +[ 0.220505 , 14 , 13 , 77 ], +[ 0.220510 , 14 , 13 , 78 ], +[ 0.221499 , 14 , 13 , 79 ], +[ 0.222785 , 13 , 12 , 35 ], +[ 0.223643 , 20 , 1 , 54 ], +[ 0.224867 , 25 , 1 , 54 ], +[ 0.226481 , 17 , 16 , 36 ], +[ 0.226609 , 36 , 2 , 83 ], +[ 0.226622 , 36 , 2 , 84 ], +[ 0.226634 , 36 , 2 , 85 ], +[ 0.226646 , 36 , 2 , 86 ], +[ 0.226659 , 36 , 2 , 87 ], +[ 0.226671 , 36 , 2 , 88 ], +[ 0.226683 , 36 , 2 , 89 ], +[ 0.226696 , 36 , 2 , 90 ], +[ 0.226708 , 36 , 2 , 91 ], +[ 0.226826 , 33 , 1 , 54 ], +[ 0.227480 , 17 , 16 , 37 ], +[ 0.227491 , 7 , 6 , 100 ], +[ 0.227496 , 7 , 6 , 101 ], +[ 0.227501 , 7 , 6 , 102 ], +[ 0.227505 , 7 , 6 , 103 ], +[ 0.227511 , 7 , 6 , 104 ], +[ 0.227515 , 7 , 6 , 105 ], +[ 0.227521 , 7 , 6 , 106 ], +[ 0.228002 , 7 , 6 , 107 ], +[ 0.228007 , 7 , 6 , 108 ], +[ 0.228012 , 7 , 6 , 109 ], +[ 0.228017 , 7 , 6 , 110 ], +[ 0.229162 , 3 , 2 , 92 ], +[ 0.229167 , 3 , 2 , 93 ], +[ 0.229172 , 3 , 2 , 94 ], +[ 0.229177 , 3 , 2 , 95 ], +[ 0.229182 , 3 , 2 , 96 ], +[ 0.230423 , 14 , 13 , 80 ], +[ 0.230435 , 14 , 13 , 81 ], +[ 0.230448 , 14 , 13 , 82 ], +[ 0.230460 , 14 , 13 , 83 ], +[ 0.230472 , 14 , 13 , 84 ], +[ 0.230485 , 14 , 13 , 85 ], +[ 0.230497 , 14 , 13 , 86 ], +[ 0.230509 , 14 , 13 , 87 ], +[ 0.230522 , 14 , 13 , 88 ], +[ 0.230534 , 14 , 13 , 89 ], +[ 0.230547 , 14 , 13 , 90 ], +[ 0.230559 , 14 , 13 , 91 ], +[ 0.230571 , 14 , 13 , 92 ], +[ 0.231547 , 14 , 13 , 93 ], +[ 0.231582 , 14 , 13 , 94 ], +[ 0.231594 , 14 , 13 , 95 ], +[ 0.231607 , 14 , 13 , 96 ], +[ 0.235105 , 9 , 8 , 17 ], +[ 0.237242 , 36 , 2 , 92 ], +[ 0.237247 , 36 , 2 , 93 ], +[ 0.237252 , 36 , 2 , 94 ], +[ 0.237257 , 36 , 2 , 95 ], +[ 0.237262 , 36 , 2 , 96 ], +[ 0.239034 , 7 , 6 , 111 ], +[ 0.239039 , 7 , 6 , 112 ], +[ 0.241063 , 14 , 13 , 97 ], +[ 0.241069 , 14 , 13 , 98 ], +[ 0.241074 , 14 , 13 , 99 ], +[ 0.241080 , 14 , 13 , 100 ], +[ 0.241086 , 14 , 13 , 101 ], +[ 0.241584 , 14 , 13 , 102 ], +[ 0.241589 , 14 , 13 , 103 ], +[ 0.242612 , 14 , 13 , 104 ], +[ 0.242617 , 14 , 13 , 105 ], +[ 0.246144 , 9 , 8 , 18 ], +[ 0.247826 , 15 , 14 , 19 ], +[ 0.247838 , 15 , 14 , 20 ], +[ 0.250063 , 7 , 6 , 113 ], +[ 0.253635 , 14 , 13 , 106 ], +[ 0.258191 , 9 , 8 , 19 ], +[ 0.261088 , 7 , 6 , 114 ], +[ 0.263517 , 17 , 16 , 38 ], +[ 0.264662 , 14 , 13 , 107 ], +[ 0.264666 , 14 , 13 , 108 ], +[ 0.269219 , 9 , 8 , 20 ], +[ 0.275699 , 14 , 13 , 109 ], +[ 0.280263 , 9 , 8 , 21 ], +[ 0.281564 , 17 , 16 , 39 ], +[ 0.286749 , 14 , 13 , 110 ], +[ 0.287828 , 15 , 14 , 21 ], +[ 0.288581 , 17 , 16 , 40 ], +[ 0.292811 , 15 , 14 , 22 ], +[ 0.294305 , 9 , 8 , 22 ], +[ 0.299330 , 5 , 4 , 11 ], +[ 0.303492 , 22 , 4 , 11 ], +[ 0.305331 , 9 , 8 , 23 ], +[ 0.307410 , 38 , 4 , 11 ], +[ 0.311379 , 5 , 4 , 12 ], +[ 0.315542 , 22 , 4 , 12 ], +[ 0.317363 , 9 , 8 , 24 ], +[ 0.319460 , 38 , 4 , 12 ], +[ 0.321204 , 10 , 9 , 12 ], +[ 0.321214 , 10 , 9 , 13 ], +[ 0.321223 , 10 , 9 , 14 ], +[ 0.321232 , 10 , 9 , 15 ], +[ 0.321241 , 10 , 9 , 16 ], +[ 0.321250 , 10 , 9 , 17 ], +[ 0.321449 , 11 , 10 , 12 ], +[ 0.321458 , 11 , 10 , 13 ], +[ 0.321468 , 11 , 10 , 14 ], +[ 0.321476 , 11 , 10 , 15 ], +[ 0.321484 , 11 , 10 , 16 ], +[ 0.321492 , 11 , 10 , 17 ], +[ 0.322204 , 10 , 9 , 18 ], +[ 0.322216 , 10 , 9 , 19 ], +[ 0.322228 , 10 , 9 , 20 ], +[ 0.322449 , 11 , 10 , 18 ], +[ 0.322454 , 11 , 10 , 19 ], +[ 0.322467 , 11 , 10 , 20 ], +[ 0.322479 , 11 , 10 , 21 ], +[ 0.323207 , 10 , 9 , 21 ], +[ 0.323452 , 11 , 10 , 22 ], +[ 0.329399 , 9 , 8 , 25 ], +[ 0.329881 , 15 , 14 , 23 ], +[ 0.330880 , 15 , 14 , 24 ], +[ 0.331878 , 15 , 14 , 25 ], +[ 0.332442 , 5 , 4 , 13 ], +[ 0.332879 , 15 , 14 , 26 ], +[ 0.336604 , 22 , 4 , 13 ], +[ 0.340440 , 9 , 8 , 26 ], +[ 0.340522 , 38 , 4 , 13 ], +[ 0.345499 , 5 , 4 , 14 ], +[ 0.348623 , 17 , 16 , 41 ], +[ 0.349661 , 22 , 4 , 14 ], +[ 0.349955 , 15 , 14 , 27 ], +[ 0.352492 , 9 , 8 , 27 ], +[ 0.353579 , 38 , 4 , 14 ], +[ 0.354651 , 17 , 16 , 42 ], +[ 0.358940 , 15 , 14 , 28 ], +[ 0.360631 , 17 , 16 , 43 ], +[ 0.364488 , 9 , 8 , 28 ], +[ 0.375534 , 9 , 8 , 29 ], +[ 0.386573 , 9 , 8 , 30 ], +[ 0.386687 , 17 , 16 , 44 ], +[ 0.389258 , 10 , 9 , 22 ], +[ 0.389271 , 10 , 9 , 23 ], +[ 0.389549 , 11 , 10 , 23 ], +[ 0.394968 , 15 , 14 , 29 ], +[ 0.397603 , 9 , 8 , 31 ], +[ 0.399717 , 17 , 16 , 45 ], +[ 0.402722 , 17 , 16 , 46 ], +[ 0.408635 , 9 , 8 , 32 ], +[ 0.408958 , 15 , 14 , 30 ], +[ 0.417770 , 17 , 16 , 47 ], +[ 0.418766 , 17 , 16 , 48 ], +[ 0.420669 , 9 , 8 , 33 ], +[ 0.431700 , 9 , 8 , 34 ], +[ 0.443731 , 9 , 8 , 35 ], +[ 0.444978 , 15 , 14 , 31 ], +[ 0.454760 , 9 , 8 , 36 ], +[ 0.455946 , 10 , 9 , 24 ], +[ 0.455958 , 10 , 9 , 25 ], +[ 0.456215 , 11 , 10 , 24 ], +[ 0.456227 , 11 , 10 , 25 ], +[ 0.464986 , 15 , 14 , 32 ], +[ 0.466797 , 9 , 8 , 37 ], +[ 0.478832 , 9 , 8 , 38 ], +[ 0.479789 , 17 , 16 , 49 ], +[ 0.484058 , 15 , 14 , 33 ], +[ 0.484067 , 15 , 14 , 34 ], +[ 0.485063 , 15 , 14 , 35 ], +[ 0.486658 , 15 , 14 , 36 ], +[ 0.489073 , 15 , 14 , 37 ], +[ 0.489868 , 9 , 8 , 39 ], +[ 0.500907 , 9 , 8 , 40 ], +[ 0.512941 , 9 , 8 , 41 ], +[ 0.522148 , 15 , 14 , 38 ], +[ 0.522617 , 10 , 9 , 26 ], +[ 0.522629 , 10 , 9 , 27 ], +[ 0.522673 , 16 , 15 , 6 ], +[ 0.522894 , 11 , 10 , 26 ], +[ 0.522906 , 11 , 10 , 27 ], +[ 0.541144 , 15 , 14 , 39 ], +[ 0.550187 , 16 , 15 , 7 ], +[ 0.550370 , 16 , 15 , 8 ], +[ 0.556560 , 5 , 4 , 15 ], +[ 0.558045 , 16 , 15 , 9 ], +[ 0.560722 , 22 , 4 , 15 ], +[ 0.564640 , 38 , 4 , 15 ], +[ 0.576158 , 15 , 14 , 40 ], +[ 0.589274 , 10 , 9 , 28 ], +[ 0.589286 , 10 , 9 , 29 ], +[ 0.589551 , 11 , 10 , 28 ], +[ 0.589563 , 11 , 10 , 29 ], +[ 0.597650 , 16 , 15 , 10 ], +[ 0.606261 , 15 , 14 , 41 ], +[ 0.606270 , 15 , 14 , 42 ], +[ 0.608275 , 15 , 14 , 43 ], +[ 0.608643 , 16 , 15 , 11 ], +[ 0.609273 , 15 , 14 , 44 ], +[ 0.624328 , 15 , 14 , 45 ], +[ 0.628820 , 17 , 16 , 50 ], +[ 0.635577 , 5 , 4 , 16 ], +[ 0.636544 , 9 , 8 , 42 ], +[ 0.639739 , 22 , 4 , 16 ], +[ 0.642609 , 16 , 15 , 12 ], +[ 0.643657 , 38 , 4 , 16 ], +[ 0.646622 , 5 , 4 , 17 ], +[ 0.648590 , 9 , 8 , 43 ], +[ 0.650784 , 22 , 4 , 17 ], +[ 0.654702 , 38 , 4 , 17 ], +[ 0.655945 , 10 , 9 , 30 ], +[ 0.655957 , 10 , 9 , 31 ], +[ 0.656238 , 11 , 10 , 30 ], +[ 0.656250 , 11 , 10 , 31 ], +[ 0.658655 , 5 , 4 , 18 ], +[ 0.660614 , 9 , 8 , 44 ], +[ 0.662817 , 22 , 4 , 18 ], +[ 0.664350 , 15 , 14 , 46 ], +[ 0.666735 , 38 , 4 , 18 ], +[ 0.669689 , 5 , 4 , 19 ], +[ 0.672651 , 9 , 8 , 45 ], +[ 0.673671 , 16 , 15 , 13 ], +[ 0.673851 , 22 , 4 , 19 ], +[ 0.677769 , 38 , 4 , 19 ], +[ 0.679638 , 16 , 15 , 14 ], +[ 0.681716 , 5 , 4 , 20 ], +[ 0.684357 , 15 , 14 , 47 ], +[ 0.684682 , 9 , 8 , 46 ], +[ 0.685879 , 22 , 4 , 20 ], +[ 0.689797 , 38 , 4 , 20 ], +[ 0.693754 , 5 , 4 , 21 ], +[ 0.697724 , 9 , 8 , 47 ], +[ 0.697916 , 22 , 4 , 21 ], +[ 0.698735 , 9 , 8 , 48 ], +[ 0.701834 , 38 , 4 , 21 ], +[ 0.704682 , 16 , 15 , 15 ], +[ 0.706649 , 16 , 15 , 16 ], +[ 0.709759 , 9 , 8 , 49 ], +[ 0.720790 , 9 , 8 , 50 ], +[ 0.722619 , 10 , 9 , 32 ], +[ 0.722631 , 10 , 9 , 33 ], +[ 0.722892 , 11 , 10 , 32 ], +[ 0.722904 , 11 , 10 , 33 ], +[ 0.726797 , 16 , 15 , 17 ], +[ 0.727835 , 16 , 15 , 18 ], +[ 0.729456 , 15 , 14 , 48 ], +[ 0.729465 , 15 , 14 , 49 ], +[ 0.729474 , 15 , 14 , 50 ], +[ 0.731460 , 15 , 14 , 51 ], +[ 0.732455 , 15 , 14 , 52 ], +[ 0.739418 , 16 , 15 , 19 ], +[ 0.746289 , 16 , 15 , 20 ], +[ 0.751407 , 15 , 14 , 53 ], +[ 0.753832 , 9 , 8 , 51 ], +[ 0.758495 , 16 , 15 , 21 ], +[ 0.765879 , 9 , 8 , 52 ], +[ 0.766737 , 16 , 15 , 22 ], +[ 0.771400 , 15 , 14 , 54 ], +[ 0.777913 , 9 , 8 , 53 ], +[ 0.781005 , 16 , 15 , 23 ], +[ 0.789288 , 10 , 9 , 34 ], +[ 0.789300 , 10 , 9 , 35 ], +[ 0.789582 , 11 , 10 , 34 ], +[ 0.789594 , 11 , 10 , 35 ], +[ 0.792949 , 9 , 8 , 54 ], +[ 0.802148 , 16 , 15 , 24 ], +[ 0.803977 , 9 , 8 , 55 ], +[ 0.816021 , 9 , 8 , 56 ], +[ 0.817929 , 16 , 15 , 25 ], +[ 0.821540 , 15 , 14 , 55 ], +[ 0.825506 , 16 , 15 , 26 ], +[ 0.831054 , 9 , 8 , 57 ], +[ 0.831411 , 15 , 14 , 56 ], +[ 0.839831 , 17 , 16 , 51 ], +[ 0.840635 , 16 , 15 , 27 ], +[ 0.842103 , 9 , 8 , 58 ], +[ 0.851474 , 15 , 14 , 57 ], +[ 0.852470 , 15 , 14 , 58 ], +[ 0.853140 , 9 , 8 , 59 ], +[ 0.853475 , 15 , 14 , 59 ], +[ 0.853484 , 15 , 14 , 60 ], +[ 0.855961 , 10 , 9 , 36 ], +[ 0.855973 , 10 , 9 , 37 ], +[ 0.856244 , 11 , 10 , 36 ], +[ 0.856257 , 11 , 10 , 37 ], +[ 0.864849 , 17 , 16 , 52 ], +[ 0.865181 , 9 , 8 , 60 ], +[ 0.873668 , 16 , 15 , 28 ], +[ 0.877226 , 9 , 8 , 61 ], +[ 0.879894 , 16 , 15 , 29 ], +[ 0.883446 , 16 , 15 , 30 ], +[ 0.890863 , 17 , 16 , 53 ], +[ 0.893281 , 9 , 8 , 62 ], +[ 0.904590 , 16 , 15 , 31 ], +[ 0.904798 , 5 , 4 , 22 ], +[ 0.905312 , 9 , 8 , 63 ], +[ 0.905875 , 16 , 15 , 32 ], +[ 0.908546 , 15 , 14 , 61 ], +[ 0.908960 , 22 , 4 , 22 ], +[ 0.912878 , 38 , 4 , 22 ], +[ 0.913547 , 15 , 14 , 62 ], +[ 0.915890 , 17 , 16 , 54 ], +[ 0.916361 , 9 , 8 , 64 ], +[ 0.921961 , 16 , 15 , 33 ], +[ 0.922644 , 10 , 9 , 38 ], +[ 0.922656 , 10 , 9 , 39 ], +[ 0.922904 , 11 , 10 , 38 ], +[ 0.922916 , 11 , 10 , 39 ], +[ 0.934286 , 16 , 15 , 34 ], +[ 0.940910 , 17 , 16 , 55 ], +[ 0.942193 , 16 , 15 , 35 ], +[ 0.944506 , 16 , 15 , 36 ], +[ 0.965936 , 17 , 16 , 56 ], +[ 0.969857 , 16 , 15 , 37 ], +[ 0.975835 , 15 , 14 , 63 ], +[ 0.975844 , 15 , 14 , 64 ], +[ 0.975863 , 15 , 14 , 65 ], +[ 0.975872 , 15 , 14 , 66 ], +[ 0.976617 , 15 , 14 , 67 ], +[ 0.982975 , 16 , 15 , 38 ], +[ 0.985992 , 16 , 15 , 39 ], +[ 0.986297 , 16 , 15 , 40 ], +[ 0.989298 , 10 , 9 , 40 ], +[ 0.989310 , 10 , 9 , 41 ], +[ 0.989594 , 11 , 10 , 40 ], +[ 0.989606 , 11 , 10 , 41 ], +[ 0.990941 , 17 , 16 , 57 ], +[ 0.995672 , 15 , 14 , 68 ], +[ 0.999683 , 15 , 14 , 69 ], +[ 1.002439 , 16 , 15 , 41 ], +[ 1.015955 , 17 , 16 , 58 ], +[ 1.024748 , 16 , 15 , 42 ], +[ 1.041958 , 17 , 16 , 59 ], +[ 1.044547 , 16 , 15 , 43 ], +[ 1.050651 , 16 , 15 , 44 ], +[ 1.050894 , 17 , 16 , 60 ], +[ 1.055988 , 10 , 9 , 42 ], +[ 1.056000 , 10 , 9 , 43 ], +[ 1.056262 , 11 , 10 , 42 ], +[ 1.056274 , 11 , 10 , 43 ], +[ 1.065697 , 15 , 14 , 70 ], +[ 1.065711 , 15 , 14 , 71 ], +[ 1.066959 , 17 , 16 , 61 ], +[ 1.075249 , 16 , 15 , 45 ], +[ 1.091945 , 16 , 15 , 46 ], +[ 1.091971 , 17 , 16 , 62 ], +[ 1.108032 , 16 , 15 , 47 ], +[ 1.116998 , 17 , 16 , 63 ], +[ 1.117068 , 16 , 15 , 48 ], +[ 1.122290 , 10 , 9 , 44 ], +[ 1.122302 , 10 , 9 , 45 ], +[ 1.122944 , 11 , 10 , 44 ], +[ 1.122957 , 11 , 10 , 45 ], +[ 1.127419 , 9 , 8 , 65 ], +[ 1.129047 , 16 , 15 , 49 ], +[ 1.129798 , 15 , 14 , 72 ], +[ 1.129807 , 15 , 14 , 73 ], +[ 1.129816 , 15 , 14 , 74 ], +[ 1.130463 , 15 , 14 , 75 ], +[ 1.130485 , 16 , 15 , 50 ], +[ 1.132780 , 15 , 14 , 76 ], +[ 1.142023 , 17 , 16 , 64 ], +[ 1.143818 , 16 , 15 , 51 ], +[ 1.146341 , 16 , 15 , 52 ], +[ 1.152832 , 15 , 14 , 77 ], +[ 1.157833 , 15 , 14 , 78 ], +[ 1.161947 , 16 , 15 , 53 ], +[ 1.162838 , 16 , 15 , 54 ], +[ 1.167045 , 17 , 16 , 65 ], +[ 1.181006 , 16 , 15 , 55 ], +[ 1.189311 , 10 , 9 , 46 ], +[ 1.189323 , 10 , 9 , 47 ], +[ 1.189328 , 16 , 15 , 56 ], +[ 1.189586 , 11 , 10 , 46 ], +[ 1.189599 , 11 , 10 , 47 ], +[ 1.190854 , 5 , 4 , 23 ], +[ 1.193063 , 17 , 16 , 66 ], +[ 1.195016 , 22 , 4 , 23 ], +[ 1.198934 , 38 , 4 , 23 ], +[ 1.199124 , 16 , 15 , 57 ], +[ 1.202835 , 16 , 15 , 58 ], +[ 1.202904 , 5 , 4 , 24 ], +[ 1.207066 , 22 , 4 , 24 ], +[ 1.210984 , 38 , 4 , 24 ], +[ 1.213944 , 5 , 4 , 25 ], +[ 1.218076 , 17 , 16 , 67 ], +[ 1.218106 , 22 , 4 , 25 ], +[ 1.218838 , 15 , 14 , 79 ], +[ 1.222024 , 38 , 4 , 25 ], +[ 1.222837 , 15 , 14 , 80 ], +[ 1.224988 , 5 , 4 , 26 ], +[ 1.225756 , 16 , 15 , 59 ], +[ 1.229150 , 22 , 4 , 26 ], +[ 1.233068 , 38 , 4 , 26 ], +[ 1.233587 , 16 , 15 , 60 ], +[ 1.237033 , 5 , 4 , 27 ], +[ 1.240916 , 16 , 15 , 61 ], +[ 1.241195 , 22 , 4 , 27 ], +[ 1.243078 , 17 , 16 , 68 ], +[ 1.245113 , 38 , 4 , 27 ], +[ 1.245397 , 16 , 15 , 62 ], +[ 1.248080 , 5 , 4 , 28 ], +[ 1.250915 , 15 , 14 , 81 ], +[ 1.250925 , 15 , 14 , 82 ], +[ 1.250934 , 15 , 14 , 83 ], +[ 1.251448 , 15 , 14 , 84 ], +[ 1.252242 , 22 , 4 , 28 ], +[ 1.253035 , 17 , 16 , 69 ], +[ 1.253897 , 15 , 14 , 85 ], +[ 1.255826 , 11 , 10 , 48 ], +[ 1.255838 , 11 , 10 , 49 ], +[ 1.255983 , 10 , 9 , 48 ], +[ 1.255995 , 10 , 9 , 49 ], +[ 1.256160 , 38 , 4 , 28 ], +[ 1.259121 , 5 , 4 , 29 ], +[ 1.263283 , 22 , 4 , 29 ], +[ 1.264721 , 16 , 15 , 63 ], +[ 1.267201 , 38 , 4 , 29 ], +[ 1.268084 , 17 , 16 , 70 ], +[ 1.268970 , 16 , 15 , 64 ], +[ 1.270157 , 5 , 4 , 30 ], +[ 1.274319 , 22 , 4 , 30 ], +[ 1.278237 , 38 , 4 , 30 ], +[ 1.280945 , 16 , 15 , 65 ], +[ 1.282198 , 5 , 4 , 31 ], +[ 1.286360 , 22 , 4 , 31 ], +[ 1.288148 , 16 , 15 , 66 ], +[ 1.290278 , 38 , 4 , 31 ], +[ 1.293092 , 17 , 16 , 71 ], +[ 1.293232 , 5 , 4 , 32 ], +[ 1.297394 , 22 , 4 , 32 ], +[ 1.301312 , 38 , 4 , 32 ], +[ 1.303023 , 16 , 15 , 67 ], +[ 1.304276 , 5 , 4 , 33 ], +[ 1.307395 , 16 , 15 , 68 ], +[ 1.308438 , 22 , 4 , 33 ], +[ 1.309955 , 15 , 14 , 86 ], +[ 1.309967 , 15 , 14 , 87 ], +[ 1.312356 , 38 , 4 , 33 ], +[ 1.316311 , 5 , 4 , 34 ], +[ 1.318101 , 17 , 16 , 72 ], +[ 1.320473 , 22 , 4 , 34 ], +[ 1.322583 , 11 , 10 , 50 ], +[ 1.322589 , 11 , 10 , 51 ], +[ 1.322601 , 11 , 10 , 52 ], +[ 1.322669 , 10 , 9 , 50 ], +[ 1.322681 , 10 , 9 , 51 ], +[ 1.324145 , 16 , 15 , 69 ], +[ 1.324391 , 38 , 4 , 34 ], +[ 1.325865 , 16 , 15 , 70 ], +[ 1.327344 , 5 , 4 , 35 ], +[ 1.331507 , 22 , 4 , 35 ], +[ 1.335425 , 38 , 4 , 35 ], +[ 1.338373 , 5 , 4 , 36 ], +[ 1.342536 , 22 , 4 , 36 ], +[ 1.344101 , 17 , 16 , 73 ], +[ 1.344808 , 16 , 15 , 71 ], +[ 1.346454 , 38 , 4 , 36 ], +[ 1.350858 , 16 , 15 , 72 ], +[ 1.366744 , 16 , 15 , 73 ], +[ 1.369122 , 17 , 16 , 74 ], +]; + +sfr_names=["http_get", + "http_post", + "https", + "http_browsing", + "exchange", + "mail_pop", + "mail_pop_1", + "mail_pop_2", + "oracle_0", + "rtp_160k", + "rtp_250k", + "smtp_0", + "smtp_1", + "smtp_2", + "video_call", + "sip_video_call_full", + "citrix_0", + "dns_0"]; + +chart("#chart4",sfr_data,sfr_names,"time-sec","flow-id"); +</script> ++++++++++++++++++++++++++++++++++ +endif::backend-docbook[] + + +=== TRex command line + +TRex commands typically include the following main arguments, but only `-f` and `-d` are required. + +[source,bash] +---- +$.sudo /t-rex-64 -f [traffic_yaml] -m [muti] -d [duration] -l [Hz=1000] -c [cores] +---- + +*-f=TRAFIC_YAML_FILE*:: + YAML traffic configuration file. + +*-m=MUL*:: + Factor for bandwidth (multiplies the CPS of each template by this value). + +*-d=DURATION*:: + Duration of the test (sec). Default: 0 + +*-l=HZ*:: + Rate (Hz) for running the latency daemon. Example: -l 1000 runs 1000 pkt/sec from each interface. A value of zero (0) disables the latency check. + +*-c=CORES*:: + Number of cores. Use 4 for TRex 40Gb/sec. Monitor the CPU% of TRex - it should be ~50%. + + +The full reference can be found xref:cml-line[here] + +==== TRex command line examples + +.Simple HTTP 1Gb/sec for 100 sec +[source,bash] +---- +$.sudo /t-rex-64 -f cap2/simple_http.yaml -c 4 -m 100 -d 100 +---- + +.Simple HTTP 1Gb/sec with latency for 100 sec +[source,bash] +---- +$.sudo /t-rex-64 -f cap2/simple_http.yaml -c 4 -m 100 -d 100 -l 1000 +---- + +.SFR 35Gb/sec traffic +[source,bash] +---- +$.sudo /t-rex-64 -f avl/sfr_delay_10_1g.yaml -c 4 -m 35 -d 100 -p +---- + +.SFR 20Gb/sec traffic with latency +[source,bash] +---- +$.sudo /t-rex-64 -f avl/sfr_delay_10_1g.yaml -c 4 -m 20 -d 100 -l 1000 +---- + +.SFR ipv6 20Gb/sec traffic with latency +[source,bash] +---- +$.sudo /t-rex-64 -f avl/sfr_delay_10_1g_no_bundeling.yaml -c 4 -m 20 -d 100 -l 1000 --ipv6 +---- + +.Simple HTTP 1Gb/sec with NAT translation support +[source,bash] +---- +$.sudo /t-rex-64 -f cap2/simple_http.yaml -c 4 -m 100 -d 100 -l 1000 --learn +---- + +.IMIX 1G/sec ,1600 flows +[source,bash] +---- +$.sudo /t-rex-64 -f cap2/imix_fast_1g.yaml -c 4 -m 1 -d 100 -l 1000 +---- + +.IMIX 1Gb/sec,100K flows +[source,bash] +---- +$.sudo /t-rex-64 -f cap2/imix_fast_1g_100k.yaml -c 4 -m 1 -d 100 -l 1000 +---- + +.64bytes ~1Gb/sec,1600 flows +[source,bash] +---- +$.sudo /t-rex-64 -f cap2/imix_64.yaml -c 4 -m 1 -d 100 -l 1000 +---- + +=== Traffic profiles provided with the TRex package + + +[options="header",cols="1,3",width="100%"] +|================= +| name | description +| cap2/dns.yaml | simple dns pcap file +| cap2/http_simple.yaml | simple http cap file +| avl/sfr_delay_10_1g_no_bundeling.yaml | sfr traffic profile capture from Avalanche - Spirent without bundeling support with RTT=10msec ( a delay machine), this can be used with --ipv6 and --learn mode +| avl/sfr_delay_10_1g.yaml | head-end sfr traffic profile capture from Avalanche - Spirent with bundeling support with RTT=10msec ( a delay machine), it is normalized to 1Gb/sec for m=1 +| avl/sfr_branch_profile_delay_10.yaml | branch sfr profile capture from Avalanche - Spirent with bundeling support with RTT=10msec it, is normalized to 1Gb/sec for m=1 +| cap2/imix_fast_1g.yaml | imix profile with 1600 flows normalized to 1Gb/sec. +| cap2/imix_fast_1g_100k_flows.yaml | imix profile with 100k flows normalized to 1Gb/sec. +| cap2/imix_64.yaml | 64byte UDP packets profile +|======================== + + +=== Stateless traffic generation + +With this feature you can "repeat" flows and create stateless, *IXIA* like streams. +After injecting the number of flows defined by `limit`, TRex repeats the same flows. If all template has a `limit` the CPS will be zero after a time as there are no new flows after the first iteration. + +*IMIX support:*:: +Example: +[source,bash] +---- +$sudo ./t-rex-64 -f cap2/imix_64.yaml -d 1000 -m 40000 -c 4 -p +---- + +[WARNING] +===================================================================== +The *-p* is used here to send the client side packets from both interfaces. +(Normally it is sent only from client ports only.) +Typically, the traffic client side is sent from the TRex client port; with this option, the port is selected by the client IP. +All the flow packets are sent from the same interface. This may create an issue with routing, as the client's IP will be sent from the server interface. PBR router configuration solves this issue but cannot be used in all cases. So use this `-p` option carefully. +===================================================================== + + +.imix_64.yaml +[source,python] +---- + cap_info : + - name: cap2/udp_64B.pcap + cps : 1000.0 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 1000 <1> +---- +<1> Repeats the flows in a loop, generating 1000 flows from this type. In this example, udp_64B includes only one packet. + +The cap file "cap2/udp_64B.pcap" includes only one packet of 64B. This configuration file creates 1000 flows that will be repeated as follows: +f1 , f2 , f3 .... f1000 , f1 , f2 ... +where the PPS == CPS for -m=1. In this case it will have PPS=1000 in sec for -m==1. +It is possible to mix stateless templates and stateful templates. + +.Imix YAML `cap2/imix_fast_1g.yaml` example +[source,python] +---- +- duration : 3 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.255" + servers_start : "48.0.0.1" + servers_end : "48.0.255.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 0 + udp_aging : 0 + mac : [0x0,0x0,0x0,0x1,0x0,0x00] + cap_info : + - name: cap2/udp_64B.pcap + cps : 90615 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 199 + - name: cap2/udp_576B.pcap + cps : 64725 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 199 + - name: cap2/udp_1500B.pcap + cps : 12945 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 199 + - name: cap2/udp_64B.pcap + cps : 90615 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 199 + - name: cap2/udp_576B.pcap + cps : 64725 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 199 + - name: cap2/udp_1500B.pcap + cps : 12945 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 199 +---- +The templates are duplicate here to better utilize DRAM and to get better performance. +//TBD: What exactly repeates the templates - TRex, script, ? Also, how does that better utilize DRAM. + +.Imix YAML `cap2/imix_fast_1g_100k_flows.yaml` example +[source,python] +---- +- duration : 3 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.255" + servers_start : "48.0.0.1" + servers_end : "48.0.255.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 0 + udp_aging : 0 + mac : [0x0,0x0,0x0,0x1,0x0,0x00] + cap_info : + - name: cap2/udp_64B.pcap + cps : 90615 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 16666 + - name: cap2/udp_576B.pcap + cps : 64725 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 16666 + - name: cap2/udp_1500B.pcap + cps : 12945 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 16667 + - name: cap2/udp_64B.pcap + cps : 90615 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 16667 + - name: cap2/udp_576B.pcap + cps : 64725 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 16667 + - name: cap2/udp_1500B.pcap + cps : 12945 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 16667 +---- + +The following example of a simple simulation includes 3 flows, with CPS=10. + +[source,python] +---- +$more cap2/imix_example.yaml +# +# Simple IMIX test (7x64B, 5x576B, 1x1500B) +# +- duration : 3 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.255" + servers_start : "48.0.0.1" + servers_end : "48.0.255.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 0 + udp_aging : 0 + mac : [0x0,0x0,0x0,0x1,0x0,0x00] + cap_info : + - name: cap2/udp_64B.pcap + cps : 10.0 + ipg : 10000 + rtt : 10000 + w : 1 + limit : 3 <1> + +---- +<1> Number of flows: 3 + + +[source,bash] +---- +./bp-sim-32-debug -f cap2/imix_example.yaml -o my.erf -v 3 > a.txt +---- + +.IMIX example limit=3 +[format="csv",cols="1^,2^,1^,1^,1^,2^,1^,2^,1^", options="header"] +|================= +pkt,time sec,template,fid,flow-pkt-id,client_ip,client_port,server_ip ,desc + 1 , 0.010000 , 0 , 1 , 1 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 2 , 0.210000 , 0 , 2 , 0 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 3 , 0.310000 , 0 , 3 , 0 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 4 , 0.310000 , 0 , 1 , 0 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 5 , 0.510000 , 0 , 2 , 0 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 6 , 0.610000 , 0 , 3 , 0 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 7 , 0.610000 , 0 , 1 , 0 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 8 , 0.810000 , 0 , 2 , 0 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 9 , 0.910000 , 0 , 1 , 0 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 10 , 0.910000 , 0 , 3 , 0 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 11 , 1.110000 , 0 , 2 , 0 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 12 , 1.210000 , 0 , 3 , 0 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 13 , 1.210000 , 0 , 1 , 0 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 14 , 1.410000 , 0 , 2 , 0 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 15 , 1.510000 , 0 , 1 , 0 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 16 , 1.510000 , 0 , 3 , 0 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 17 , 1.710000 , 0 , 2 , 0 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 18 , 1.810000 , 0 , 3 , 0 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 19 , 1.810000 , 0 , 1 , 0 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 20 , 2.010000 , 0 , 2 , 0 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 21 , 2.110000 , 0 , 1 , 0 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 22 , 2.110000 , 0 , 3 , 0 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 23 , 2.310000 , 0 , 2 , 0 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 24 , 2.410000 , 0 , 3 , 0 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 25 , 2.410000 , 0 , 1 , 0 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 26 , 2.610000 , 0 , 2 , 0 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 27 , 2.710000 , 0 , 1 , 0 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 28 , 2.710000 , 0 , 3 , 0 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 29 , 2.910000 , 0 , 2 , 0 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 30 , 3.010000 , 0 , 3 , 0 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 31 , 3.010000 , 0 , 1 , 0 , 16.0.0.1 , 1024 , 48.0.0.1 , -> +|================= + +* Average CPS: 10 packets per second (30 packets in 3 sec). +* Total of 3 flows, as specified in the configuration file. +* The flows come in bursts, as specified in the configuration file. + + +ifndef::backend-docbook[] ++++++++++++++++++++++++++++++++++ +<div id="chart_imix" style="font : 10px sans-serif"; ></div> + +<script> +var imix_data=[ +[ 0.010000 , 1 , 0 , 1 ], +[ 0.210000 , 2 , 0 , 0 ], +[ 0.310000 , 3 , 0 , 0 ], +[ 0.310000 , 1 , 0 , 0 ], +[ 0.510000 , 2 , 0 , 0 ], +[ 0.610000 , 3 , 0 , 0 ], +[ 0.610000 , 1 , 0 , 0 ], +[ 0.810000 , 2 , 0 , 0 ], +[ 0.910000 , 1 , 0 , 0 ], +[ 0.910000 , 3 , 0 , 0 ], +[ 1.110000 , 2 , 0 , 0 ], +[ 1.210000 , 3 , 0 , 0 ], +[ 1.210000 , 1 , 0 , 0 ], +[ 1.410000 , 2 , 0 , 0 ], +[ 1.510000 , 1 , 0 , 0 ], +[ 1.510000 , 3 , 0 , 0 ], +[ 1.710000 , 2 , 0 , 0 ], +[ 1.810000 , 3 , 0 , 0 ], +[ 1.810000 , 1 , 0 , 0 ], +[ 2.010000 , 2 , 0 , 0 ], +[ 2.110000 , 1 , 0 , 0 ], +[ 2.110000 , 3 , 0 , 0 ], +[ 2.310000 , 2 , 0 , 0 ], +[ 2.410000 , 3 , 0 , 0 ], +[ 2.410000 , 1 , 0 , 0 ], +[ 2.610000 , 2 , 0 , 0 ], +[ 2.710000 , 1 , 0 , 0 ], +[ 2.710000 , 3 , 0 , 0 ], +[ 2.910000 , 2 , 0 , 0 ], +[ 3.010000 , 3 , 0 , 0 ], +[ 3.010000 , 1 , 0 , 0 ], +]; + +chart("#chart_imix",imix_data,["udp_64"],"time-sec","flow-id"); +</script> ++++++++++++++++++++++++++++++++++ +endif::backend-docbook[] + +=== Clients/Servers IP allocation scheme + +Currently, there is one global IP pool for clients and servers. It serves all templates. all the templates will allocate IP from this global pool. +Each TRex client/server "dual-port" (pair of ports, such as port 0 for client, port 1 for server) has it own mask offset taken from the YAML. The mask offset is called `dual_port_mask`. + +Example: + +[source,python] +---- +generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.255" + servers_start : "48.0.0.1" + servers_end : "48.0.0.255" + dual_port_mask : "1.0.0.0" <1> + tcp_aging : 0 + udp_aging : 0 +---- +<1> Mask to add per dual-port pair. +The reason we introduce dual_port_mask is to make static route configurable. With this mask, different ports has different prefix. + +//TBD: needs clarification - this is the format of a port mask? + +With four ports, TRex produces the following output: + +[source,python] +---- + dual-0 (0,1) --> C (16.0.0.1-16.0.0.128 ) <-> S( 48.0.0.1 - 48.0.0.128) + dual-1 (2,3) --> C (17.0.0.129-17.0.0.255 ) <-> S( 49.0.0.129 - 49.0.0.255) + mask ("1.0.0.0") +---- + +In the case of setting dual-port_mask as 0.0.0.0, both ports will use the same range of ip. +With four ports and dual_port_mask as 0.0.0.0, the ip range is : + +[source,python] +---- + dual-0 (0,1) --> C (16.0.0.1-16.0.0.128 ) <-> S( 48.0.0.1 - 48.0.0.128) + dual-1 (2,3) --> C (16.0.0.129-16.0.0.255 ) <-> S( 48.0.0.129 - 48.0.0.255) +---- + +//TBD: not clear what the following 5 points are referring to. This looks like it should be a continuation of the footnotes for the example a few lines up. +- Number of clients : 255 +- Number of servers : 255 +- The mask defined by dual_port_mask (1.0.0.0) is added for each dual-port pair, but the total number of clients/servers from YAML will be constant and does not depend on the amount of dual ports. +- TCP/UDP aging is required when the number of clients is very small and the template defines a very long duration. +This is the time it takes to return the socket to the pool. +//TBD: not clear - is TCP/UDP aging an option used when the template defines a long duration? also, should specify what "very long" refers to. +- In the current version, the only option for distribution is "seq". + + +*Router configuration for this mode:*:: + +PBR is not necessary. The following configuration is sufficient. +//TBD: clarify + +[source,python] +---- +interface TenGigabitEthernet1/0/0 <1> + mac-address 0000.0001.0000 + mtu 4000 + ip address 11.11.11.11 255.255.255.0 +! +` +interface TenGigabitEthernet1/1/0 <2> + mac-address 0000.0001.0000 + mtu 4000 + ip address 22.11.11.11 255.255.255.0 +! +interface TenGigabitEthernet1/2/0 <3> + mac-address 0000.0001.0000 + mtu 4000 + ip address 33.11.11.11 255.255.255.0 +! +interface TenGigabitEthernet1/3/0 <4> + mac-address 0000.0001.0000 + mtu 4000 + ip address 44.11.11.11 255.255.255.0 + load-interval 30 + + +ip route 16.0.0.0 255.0.0.0 22.11.11.12 +ip route 48.0.0.0 255.0.0.0 11.11.11.12 +ip route 17.0.0.0 255.0.0.0 44.11.11.12 +ip route 49.0.0.0 255.0.0.0 33.11.11.12 +---- +<1> Connected to TRex port 0 (client side) +<2> Connected to TRex port 1 (server side) +<3> Connected to TRex port 2 (client side) +<4> Connected to TRex port 3(server side) + +*One server:*:: + +To support a template with one server, you can add a new YAML server_addr ip. Each dual-port pair will be assigned a separate server (in compliance with the mask). +//TBD: clarify + +[source,python] +---- +- name: cap2/dns.pcap + cps : 1.0 + ipg : 10000 + rtt : 10000 + w : 1 + server_addr : "48.0.0.1" <1> + one_app_server : true <2> + wlength : 1 +---- +<1> Server IPv4 address. +<2> Enable one server mode. + +*w/wlength:*:: +//TBD: looks like this should be a continuation of the footnotes as in 1 and 2 above. + +not require to configure them, user 1 +//TBD: ? + +*new statistic:*:: + +[source,python] +---- + Active-flows : 19509 Clients : 504 Socket-util : 0.0670 % + Open-flows : 247395 Servers : 65408 Socket : 21277 Socket/Clients : 42.2 +---- + + +[NOTE] +===================================================================== +* No backward compatibility with the old generator YAML format. +* When using -p option, TRex will not comply with the static route rules. Server-side traffic may be sent from the client side (port 0) and vice-versa. Use the -p option only with PBR configuration when the router, switch p1<->p2. +//TBD: "when router..." unclear +* VLAN (sub interface feature) does not comply with static route rules. Use it only with PBR. + VLAN0 <-> VALN1 per interface + vlan : { enable : 1 , vlan0 : 100 , vlan1 : 200 } +* Limitation: When using a template with plugins (bundles), the number of servers must be higher than the number of clients. +===================================================================== + +==== More Details about IP allocations + +Each time a new flow is creaed, TRex allocates a new Client IP/port and Server IP. This 3-tuple should be distinct among active flows. + +Currently, only sequcency distribution is supported in IP allocation. That means the IP address is increased one by one. + +Let's say if we have 2 candidate IPs in the pool: 16.0.0.1 and 16.0.0.2. So the sequence of allocated clients should be something like this: + +[source,python] +---- +16.0.0.0.1 [1024] +16.0.0.0.2 [1024] +16.0.0.0.1 [1025] +16.0.0.0.2 [1025] +---- + +==== How to decide the PPS and BPS + +- Example of one flow with 4 packets +- Green are first packet of flow +- Lets say the client ip pool starts from 16.0.0.1, and the distribution is seq. + +image:images/ip_allocation.png[title="rigt"] + +latexmath:[$Total PPS = \sum_{k=0}^{n}(CPS_{k}\times {flow\_pkts}_{k})$] + +latexmath:[$Concurrent flow = \sum_{k=0}^{n}CPS_{k}\times flow\_duration_k $] + + +The above fomulars can be used to calculate the PPS. The TRex throughput depends on the PPS calculated above and the value of m (a multiplier assigned by TRex cli). + +The m value is a multiplier of total pcap files CPS. +CPS of pcap file is configured on yaml file. + +Let's take a simple example as below. + + +[source,python] +---- +cap_info : + - name: avl/first.pcap < -- has 2 packets + cps : 102.0 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/second.pcap < -- has 20 packets + cps : 50.0 + ipg : 10000 + rtt : 10000 + w : 1 +---- + +The throughput is: 'm*(CPS_1*flow_pkts+CPS_2*flow_pkts)' + +So if the m is set as 1, the total PPS is : 102*2+50*20 = 1204 PPS. + +The BPS depends on the packet size. You can refer to your packet size and get the BPS = PPS*Packet_size. + +==== Roadmap of Client/Server IP allocation + +We have several features under development for IP allocation. + +- *1) per-template generator* + +Multiple generators can be defined and assigned to different pcap file templates. + +The YAML configuration is something like this: + + +[source,python] +---- + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.1.255" + servers_start : "48.0.0.1" + servers_end : "48.0.20.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 0 + udp_aging : 0 + generator_clients : + - name : "c1" + distribution : "random" + ip_start : "38.0.0.1" + ip_end : "38.0.1.255" + clients_per_gb : 201 + min_clients : 101 + dual_port_mask : "1.0.0.0" + tcp_aging : 0 + udp_aging : 0 + generator_servers : + - name : "s1" + distribution : "seq" + ip_start : "58.0.0.1" + ip_end : "58.0.1.255" + dual_port_mask : "1.0.0.0 + cap_info : + - name: avl/delay_10_http_get_0.pcap + cps : 404.52 + ipg : 10000 + rtt : 10000 + w : 1 + - name: avl/delay_10_http_post_0.pcap + client_pool : "c1" + server_pool : "s1" + cps : 404.52 + ipg : 10000 + rtt : 10000 + w : 1 +---- + +- *2) More distributions will be supported (normal distribution, random distribution, etc)* + +Currently, only sequcence is supported. + +- *3) Histogram of tuple pool will be supported* + +This feature gives user more flexibility to define the IP generator. + +[source,python] +---- + generator : + client_pools: + - name : "a" + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.1.255" + tcp_aging : 0 + udp_aging : 0 + + - name : "b" + distribution : "random" + clients_start : 26.0.0.1" + clients_end : 26.0.1.255" + tcp_aging : 0 + udp_aging : 0 + + - name : "c" + pools_list : + - name:"a" + probability: 0.8 + - name:"b" + probability: 0.2 +---- + + + +=== Measure Jitter/Latency + +To measure jitter/latency on high priorty packets (one SCTP flow), use `-l [Hz]` where Hz defines the number of packets to send from each port per second. This option measures latency and jitter in the latency. The shell output is similar to the following: + +[source,python] +---- + Cpu Utilization : 0.1 % + if| tx_ok , rx_ok , rx ,error, average , max , Jitter<1> ,max + | , , check, , latency(usec),latency (usec) ,(usec) , window + -------------------------------------------------------------------------------------- + 0 | 1002, 1002, 2501, 0, 61 , 70, 3 | 60 60 + 1 | 1002, 1002, 2012, 0, 56 , 63, 2 | 50 51 + 2 | 1002, 1002, 2322, 0, 66 , 74, 5 | 68 59 + 3 | 1002, 1002, 1727, 0, 58 , 68, 2 | 52 49 + + Rx Check stats enabled + --------------------------------------------------------------------------------------- + rx check: avg/max/jitter latency, 94 , 744, 49<1> | 252 287 3 + + active flows: 10, fif: 308, drop: 0, errors: 0 + --------------------------------------------------------------------------------------- +---- +<1> Jitter information + + diff --git a/trex_config.asciidoc b/trex_config.asciidoc new file mode 100755 index 00000000..06bf3afc --- /dev/null +++ b/trex_config.asciidoc @@ -0,0 +1,299 @@ +TRex first time configuration +============================= +:author: hhaim with the Help of Amir Kroparo +:email: <hhaim@cisco.com> +:description: TRex Getting started - instalation guide +:revdate: 2014-11-01 +:revnumber: 0.1 +:deckjs_theme: swiss +:deckjs_transition: horizontal-slide +:scrollable: + + +++++++++++++++++++ +<script type="text/javascript" + src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"> +</script> + +<script src="http://d3js.org/d3.v3.min.js" charset="utf-8"></script> + +<script src="my_chart.js"></script> + +<style> +.axis path, +.axis line { + fill: none; + stroke: #000; + shape-rendering: crispEdges; +} + +.dot { + stroke: #000; +} +</style> + + +<style type="text/css"> + +h1 { + font-size: 2.5em; +} + +h2 { + font-size: 1.5em; + color: #CD7300; + border-bottom-color: #000; +} + +h7 { + font-size: 4.5em; + color: #CD7300; + position: relative; + top: auto; + text-align: center; + padding: 0; + -webkit-transform: none; + -moz-transform: none; + -ms-transform: none; + -o-transform: none; + transform: none; + padding: 0 48px; + position: absolute; + left: 0; + right: 0; + top: 50%; +} + +h8 { + font-size: 2.25em; + font-weight: bold; + padding-top: .5em; + margin: 0 0 .66666em 0; + border-top: 3px solid #888; + color: #c00; + border-top-color: #ccc; + left: 0; + right: 0; + top: 40%; +} + + +html, body { + height: 100%; + margin: 0 auto; + max-width: 1000px; +} + +</style> + +<script> + $('#title-slide').css("background-image", "url('images/trex_logo.png')"); + $('#title-slide').css("background-repeat","no-repeat"); + $('#title-slide').css("background-position","center"); + $('h1').html(''); + $('h3').html('<font size="4">Hanoch Haim </font>'); + $('h4').html('<font size="4">04/2015</font>'); + </script> + +++++++++++++++++++ + + +== Simple configuration + +* TRex does not implement ARP emulation +* This guide will help you to configure Cisco ASR1K to work with TRex +* TRex is directly connected to ASR1K ports. + +image::images/TrexConfig.png[title="TRex/Router setup"] +. TRex port 0 - clients side +. Router TenG 0/0/0 +. Router TenG 0/0/1 +. TRex port 1 - servers side + + +== TRex configuration + +* TRex act as both client and server side +* TRex port mac addrees should configure correctly, so packet generated from port 1 will get to 2 and vice-versa +* To use the config file you can add this switch `--cfg [file]` +* Or edit the configuration file in `/etc/trex_cfg.yaml` + +[source,python] +---- + - port_limit : 2 + port_info : # set eh mac addr + - dest_mac : [0x1,0x0,0x0,0x1,0x0,0x00] <1> + src_mac : [0x2,0x0,0x0,0x2,0x0,0x00] <2> + - dest_mac : [0x3,0x0,0x0,0x3,0x0,0x00] <3> + src_mac : [0x4,0x0,0x0,0x4,0x0,0x00] <4> +---- +<1> Correspond to TRex port 0 - should be Router TenG 0/0/0 mac-address +<2> Should be distinc mac-addrees, router should be configure to sent to this mac-addrees +<3> Correspond to TRex port 0 - should be Router TenG 0/0/1 mac-address +<4> Should be distinc mac-addrees, router should be configure to sent to this mac-addrees + + +== Router configuration PBR part 1 + +* Router moves packets from port 0->1 and 1->0 without looking into IP addrees. + +* TenG 0/0/0 <-> TenG 0/0/1 + +*Router configuration:*:: + +[source,python] +---- +interface TenGigabitEthernet0/0/0 + mac-address 0100.0001.0000 <1> + mtu 4000 <2> + ip address 11.11.11.11 255.255.255.0 <3> + ip policy route-map p1_to_p2 <4> + load-interval 30 +! + +interface TenGigabitEthernet0/0/1 + mac-address 0300.0003.0000 <5> + mtu 4000 + ip address 12.11.11.11 255.255.255.0 + ip policy route-map p2_to_p1 + load-interval 30 +! +---- +<1> Configure mac-addrees to match TRex destination port-0 +<2> Set MTU +<3> Set an ip addrees ( routing can't work without this) +<4> Configure PBR policy - see next slide +<5> Configure mac-addrees to match TRex destination port-1 + +== Router configuration PBR part 2 + +[source,python] +---- + +route-map p1_to_p2 permit 10 + set ip next-hop 12.11.11.12 <1> +! +route-map p2_to_p1 permit 10 + set ip next-hop 11.11.11.12 <2> + +---- + +<1> Set the destination packet to be 12.11.11.12 which correspond to TenG 0/0/1 +<2> Set the destination packet to be 11.11.11.12 which correspond to TenG 0/0/0 + + +== Router configuration PBR part 3 + +* What about destination mac-address it should be TRex source mac-addrees? +* The folowing configuration addrees it + +[source,python] +---- + arp 11.11.11.12 0200.0002.0000 ARPA <1> + arp 12.11.11.12 0400.0004.0000 ARPA <2> +---- +<1> Destination mac-addrees of packets sent from If 0/0/0 is matched to TRex source mac-address port-0 +<2> Destination mac-addrees of packets sent from If 0/0/1 is matched to TRex source mac-address port-1 + +== Static-route configuration - TRex + +* You can set static range of IPs for client and server side + +[source,python] +---- +generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.255" + servers_start : "48.0.0.1" + servers_end : "48.0.0.240" + dual_port_mask : "1.0.0.0" + tcp_aging : 0 + udp_aging : 0 +---- + +* In this example, you should expect: +** Number of clients 255 +** Number of servers 240 + +== Static-route configuration - Router + +[source,python] +---- +interface TenGigabitEthernet0/0/0 + mac-address 0100.0001.0000 + mtu 4000 + ip address 11.11.11.11 255.255.255.0 +! +` +interface TenGigabitEthernet0/0/1 + mac-address 0300.0003.0000 + mtu 4000 + ip address 22.11.11.11 255.255.255.0 +! +ip route 16.0.0.0 255.0.0.0 22.11.11.12 <1> +ip route 48.0.0.0 255.0.0.0 11.11.11.12 <2> +---- +<1> Match the range of TRex YAML ( client side 0/0/0 ) +<2> Match the range of TRex YAML ( server side 0/0/1) + +== Verify configuration + +* To verify that TRex port-0 is connected to Router 0/0/0 and not 0/0/1 run + +........................................... +$./t-rex-64 -f cap2/dns.yaml -m 1 -d 100 -l 1000 --lo --lm 1 +........................................... +* It sends packets only from TRex port-0 ( `--lm 1` ) + + +* to send only from TRex port 1 do this: +........................................... +$./t-rex-64 -f cap2/dns.yaml -m 1 -d 100 -l 1000 --lo --lm 2 +........................................... + + +== Static-route configuration - IPV6 + +[source,python] +---- +interface TenGigabitEthernet1/0/0 + mac-address 0000.0001.0000 + mtu 4000 + ip address 11.11.11.11 255.255.255.0 + ip policy route-map p1_to_p2 + load-interval 30 + ipv6 enable #<1> + ipv6 address 2001:DB8:1111:2222::1/64 #<2> + ipv6 policy route-map ipv6_p1_to_p2 #<3> +! + + +ipv6 unicast-routing #<4> + +ipv6 neighbor 3001::2 TenGigabitEthernet0/1/0 0000.0002.0002 #<5> +ipv6 neighbor 2001::2 TenGigabitEthernet0/0/0 0000.0003.0002 + +route-map ipv6_p1_to_p2 permit 10 #<6> + set ipv6 next-hop 2001::2 +! +route-map ipv6_p2_to_p1 permit 10 + set ipv6 next-hop 3001::2 +! + + +csi-mcp-asr1k-40(config)#ipv6 route 4000::/64 2001::2 +csi-mcp-asr1k-40(config)#ipv6 route 5000::/64 3001::2 +---- +<1> Enable ipv6 +<2> Add ipv6 addrees +<3> Add pbr +<4> Enable ipv6 routing +<5> Mac-addr setting should be like T-Rex +<6> PBR configuraion + + + + + + diff --git a/trex_control_plane_design_phase1.asciidoc b/trex_control_plane_design_phase1.asciidoc new file mode 100755 index 00000000..cbd02670 --- /dev/null +++ b/trex_control_plane_design_phase1.asciidoc @@ -0,0 +1,516 @@ +T-Rex Control Plane Design - Phase 1 +==================================== +:author: Dan Klein +:email: <danklei@cisco.com> +:revnumber: 1.0 +:quotes.++: +:numbered: + + +== Introduction + +=== T-Rex traffic generator + +T-Rex traffic generator is a tool design the benchmark platforms with realistic traffic. +This is a work-in-progress product, which is under constant developement, new features are added and support for more router's fuctionality is achieved. + +=== T-Rex Control Plane + +T-Rex control (phase 1) is the base API, based on which any future API will be developed. + +This document will describe the current control plane for T-Rex, and its scalable features as a directive for future developement. + +==== T-Rex Control Plane - Architecture and Deployment notes + +T-Rex control plane is based on a JSON RPC transactions between clients and server. + +Each T-Rex machine will have a server running on it, closely interacting with T-Rex (clients do not approach T-Rex directly). + +The server version (which runs as either a daemon or a CLI application) is deployed with T-Rex latest version, written in Python 2.7. +As future feature, and as multiple T-Rexes might run on the same machine, single server shall serve all T-Rexes running a machine. + +The control plane implementation is using the currently dumped data messaging from T-Rex's core via ZMQ publisher, running from core #1. +The server used as a Subscriptor for this data, manipulating the packets, and re-encodes it into JSON-RPC format for clients use. + +Since the entire process is taken place internally on the machine itself (using TCP connection with `localhost`), very little overhead is generated from outer network perspective. + +<<< + +The following image describes the general architecture of the control plane and how it interacts with the data plane of T-Rex. + +ifdef::backend-docbook[] +image::images/trex_control_plane_modules.png[title="Control Plane modules",align="center",width=450, link="images/trex_control_plane_modules.png"] +endif::backend-docbook[] + +ifdef::backend-xhtml11[] +image::images/trex_control_plane_modules.png[title="Control Plane modules",align="center",width=900, link="images/trex_control_plane_modules.png"] +endif::backend-xhtml11[] + +The Python test script block represents any automation code or external module that wishes to control T-Rex by interacting with its server. + +Such script can use other JSON-RPC based implementations of this CTRexClient module, as long as it corresponds with the known server methods and JSON-RPC protocol. + +At next phases, an under developement integrated module will serve the clients, hence eliminating even the internal TCP messaging on the machine footnote:[updating server side planned to have almost no affect on the client side]. + +== Using the API + +[NOTE] +Basic familiarity with T-Rex is recommended before using this tool. + +Further information can be learned from T-Rex manual: http://csi-wiki-01:8080/display/bpsim/Documentation[(T-Rex manual)] + + +=== The Server module + +The server module is responsible for handling all possible requests related to T-Rex (i.e. this is the only mechanism that interacts with remote clients). + +The server is built as a multithreaded application, and **must be launched on a T-Rex commands using `sudo` permissions**. + +The server application can run in one of two states: + + 1. **Live monitor**: this will run the server with live logging on the screen. To launch the server in this mode run `server/trex_server.py` file directly. + + 2. **Daemon application**: this will run the server as a background daemon process, and all logging will be saved into file, located at `/var/log/trex/` path. + + This is the common scenario, during which nothing is prompted into the screen, unless in case of error in server launching. + +==== Launching the server + +The server would run only on valid T-Rex machines or VM, due to delicate customization in used sub-modules, designed to eliminate the situation in which control and data plane packets are mixed. + +The server code is deployed by default with T-Rex (starting version 1.63 ) and can be launched from its path using the following command: + +`./trex_daemon_server [RUN_COMMAND] [options]` + +[NOTE] +The [RUN_COMMAND] is used only when server launched as a daemon application. + +Running this command with `--help` option will prompt the help menu, explaning all the available options. + +===== Daemon commands + +The following daemon commands are supported: + + 1. **`start`**: This option starts the daemon application of T-Rex server, using the following command options (detailed exmplanation on this next time). + + 2. **`stop`**: Stop the daemon application. + + 3. **`restart`**: Stop the current daemon proccess, then relaunch it with the provided parameters (the parameters must be re-entered). + + 3. **`show`**: Prompt whether the daemon is running or not. + +WARNING: restarting the daemon application will **truncate** the logfile. + +===== Server options commands + +The following describes the options for server launching, and applies to both daemon and live launching. + +Let's have a look on the help menu: + +---- +[root@trex-dan Server]# ./trex_daemon_server --help +[root@trex-dan Server]# usage: trex_deamon_server {start|stop|restart} [options] + + NOTE: start/stop/restart options only available when running in daemon mode + +Run server application for T-Rex traffic generator + +optional arguments: + -h, --help show this help message and exit + -p PORT, --daemon-port PORT + Select port on which the daemon runs. Default port is + 8090. + -z PORT, --zmq-port PORT + Select port on which the ZMQ module listens to T-Rex. + Default port is 4500. #<2> + -t PATH, --trex-path PATH + Specify the compiled T-Rex directory from which T-Rex + would run. Default path is: / #<1> + +[root@trex-dan Server]# +---- + +<1> Default path might change when launching the server in daemon or live mode. + +<2> ZMQ port must match the defined port of the platform, generally found at `/etc/trex_cfg.yaml`. + +The available options are: + + 1. **`-p, --daemon-port`**: set the port on which the server is listening to clients requests. + + Default listening server port is **`8090`**. + + 2. **`-z, --zmq-port`**: set the port on which the server is listening to zmq publication from T-Rex. + + Default listening server port is **`4500`**. + + 3. **`-t, --trex-path`**: set the path from which T-Rex is runned. This is especially helpful when more than one version of T-Rex is used or switched between. Although this field has default value, it is highly recommended to set it manually with each server launch. + +[NOTE] +When server is launched is first makes sure the trex-path is valid: the path 'exists' and granted with 'execution permissions.' If any of the conditions is not valid, the server will not launch. + + +=== The Client module + +The client is a Python based application that created `TRexClient` instances. + +Using class methods, the client interacts with T-Rex server, and enable it to perform the following commands: + + 1. Start T-Rex run (custom parameters supported). + + 2. Stop T-Rex run. + + 3. Check what is the T-Rex status (possible states: `Idle, Starting, Running`). + + 4. Poll (by customize sampling) the server and get live results from T-Rex **while still running**. + + 5. Get custom T-Rex stats based on a window of saved history of latest 'N' polling results. + +The clients is also based on Python 2.7, however unlike ther server, it can run on any machine who wishes to. + +In fact, the client side is simply a python library that interact with the werver using JSON-RPC (v2), hence if needed, anyone can write a library on any other language that will interact with the server ins the very same way. + + +==== `CTRexClient` module initialization + +As explained, `CTRexClient` is the main module to use when writing an T-Rex test-plan. + +This module holds the entire interaction with T-Rex server, and result contiaing via `result_obj`, which is an instance of `CTRexResult` class. + +The `CTRexClient` instance is initialized in the following way: + + 1. **T-Rex hostname**: represents the hostname on which the server is listening. Both hostname or IPv4 address will be a valid input. + + 2. **Server port**: the port on which the server listens to incoming client requests. This parameter value must be identical to `port` option configured in the server. + + 3. **History size**: The number of saved T-Rex samples. Based on this "window", some extra statistics and data are calculated. Default history size is 100 samples. + + 4. **verbose **: This boolean option will prompt extended output, if available, of each of the activated methods. For any method that interacts with T-Rex server, this will prompt the JSON-RPC request and response. + + This option is especially useful for developers who wishes to imitate the functionality of this client using other programming languages. + +**That's it!** + +Once these parameter has been passed, you're ready to interact woth T-Rex. + +[NOTE] +The most common initialization will simply use the hostname, such that common initilization lookes like: + +`trex = CTRexClient('trex_host_name')` + +==== `CTRexClient` module usage + +This section covers with great detail the usage of the client module. Each of the methods describes are class methods of `CTRexClient`. + + - **`start_trex (f, d, block_to_success, timeout, trex_cmd_options)`** + + Issue a request to start T-Rex with certain configuration. The server will only handle the request if the T-Rex is in `Idle` status. + + Once the status has been confirmed, T-Rex server will issue for this single client a token, so that only that client may abort running T-Rex session. + + `f` and `d` parameters are mandatory, as they are crutial parameter in setting T-Rex bahviour. Also, `d` parameter must be at least 30 seconds or larger. + By default (and by design) this method **blocks** until T-Rex status changes to either 'Running' or back to 'Idle'. + + - **`stop_trex()`** + + If (and only if) a certain client issued a run requested (and it accepted), this client may use this command to abort current run. + + This option is very useful especially when the real-time data from the T-Rex are utilized. + + - **`wait_until_kickoff_finish(timeout = 40)`** + + This method blocks until T-Rex T-Rex status changes to 'Running'. In case of error an exception will be thrown. + + The 'timeout' parameter sets the maximum waiting time. + + This method is especially useful when `block_to_success` was set to false in order to utilize the time to configure other things, such as DUT. + + - **`is_running(dump_out = False)`** + + Checks if there's currently T-Rex session up (with any client). + + If T-Rex is running, this method returns `True` and the result object id updated accordingly. + + If not running, return `False`. + + If a dictionary pointer is given in `dump_out` argument, the pointer object is cleared and the latest dump stored in it. + + - **`get_running_status()`** + + Fetches the current T-Rex status. + + Three possible states + + * `Idle` - No T-Rex session is currently running. + + * `Starting` - A T-Rex session just started (turns into Running after stability condition is reached) + + * `Running` - T-Rex session is currently active. + + The following diagram describes the state machine of T-Rex: + +ifdef::backend-docbook[] +image::images/trex_control_plane_state_machine.png[title="T-Rex running state machine",align="center",width=280, link="images/trex_control_plane_state_machine.png"] +endif::backend-docbook[] + +ifdef::backend-xhtml11[] +image::images/trex_control_plane_state_machine.png[title="T-Rex running state machine",align="center",width=400, link="images/trex_control_plane_state_machine.png"] +endif::backend-xhtml11[] + + - **`get_running_info()`** + + This method performs single poll of T-Rex running data and process it into the result object (named `result_obj`). + + The method returns the most updated data dump from T-Rex in the form of Python dictionary. + + + + Behind the scenes, running that method will trigger inner-client process over the saved window, and produce window-relevant inforamtion, as well as get the most important data moe accessible. + + Once the data has been fetched (at sample rate the satisfies the user), a custom data manipulation can be done in various forms and techniques footnote:[See `CTRexResult` module usage for more details]. + + **Note: ** the sampling rate is bounded from buttom to 2 samples/sec. + + - **`sample_until_condition(condition_func, time_between_samples = 5)`** + + This method automatically sets ongoing sampling of T-Rex data, with sampling rate described by `time_between_samples`. On each fetched dump, the `condition_func` is applied on the result objects, and if returns `True`, the sampling will stop. + + On success (condition has been met), this method returns the latest result object that satisfied the given condition. + + ON fail, this method will raise `UserWarning` exception. + + - **`sample_to_run_finish(time_between_samples = 5)`** + + This method automatically sets ongoing sampling of T-Rex data with sampling rate described by `time_between_samples` until T-Rex run finished. + + - **`get_result_obj()`** + + Returns a pointer to the result object of the client instance. + + Hence, this method returns the result object on which all the data processing takes place. + +TIP: The window stats (calculated when `get_running_info()` triggered) are very helpful in eliminate 'spikes' behavior in numerical values which might float from other data. + +==== `CTRexResult` module usage + +This section covers how to use `CTRexResult` module to access into T-Rex data and post processing results, taking place at the client side whenever a data is polled from the server. + +The most important data structure in this module is the `history` object, which contains the sampled information (plus the post processing step) of each sample. + +Most of the class methods are getters that enables an easy access to the most commonly used when working with T-Rex. These getters are called with self-explained names, such as `get_max_latency`. + +However, on top to these methods, the class offers data accessibility using the rest of the class methods. + +These methods are: + + - **`is_done_warmup()`** + + This will return `True` only if T-Rex has reached its expected transmission bandwidth footnote:[A 3% deviation is allowed.]. + + This parameter is important since in most cases, the most relevent test cases are interesting when T-Rex produces its expected TX, based on which the platform is tested and benchmerked. + + - **`get_latest_dump()`** + + Fetches the latest polled dump saved in history. + + - **`get_last_value (tree_path_to_key, regex = None)`** + + Fetch, out of the latest data dump a value. + + - **`get_value_list (tree_path_to_key, regex = None)`** + + Fetch, out of all data dumps stored in history a value. + + - **History data access API** + + Since (as metioned earlier) the data dump is a JSON-RPC string, which is decoded into Python dictionaries and lists, nested within each other. + + This "Mini API" is used by both `get_last_value` and `get_value_list` methods, and receives in both cases two arguments: `tree_path_to_key, regex` footnote:[By default, `regex` argument is set to None]. + + The user may choose whatever value he wishes to extract, using the `tree_path_to_key` argument. + + * In order to get deeper and deeper on the hierarchy, use the key of the dictionary, separated by dot (‘'.'’) for each level. + + In order to fetch more than one key in a certain dictionary (no matter how deep it is nested), use the `regex` argument to state which keys are to be included. + Example: In order to fetch only the `expected_tx` key values of the latest dump, we'll call: *`get_last_value("trex-global.data", "m_tx_expected_\w+")`* + + This will produce the following dictionary result: + + *`{'m_tx_expected_pps': 21513.6, 'm_tx_expected_bps': 100416760.0, 'm_tx_expected_cps': 412.3}`* + + We can see that the result is every key-value pair, found at the relevant tree-path and matches the provided regex. + + * In order to access an array element, specifying the `key_to_array[i]`, where `i` is the desired array index. + + Example: In order to access the third element of the data array of: + + `{“template_info” : {"name":"template_info","type":0,"data":["avl/delay_10_http_get_0.pcap","avl/delay_10_http_post_0.pcap",` *`"avl/delay_10_https_0.pcap"`* `,"avl/delay_10_http_browsing_0.pcap", "avl/delay_10_exchange_0.pcap","avl/delay_10_mail_pop_0.pcap","avl/delay_10_mail_pop_1.pcap","avl/delay_10_mail_pop_2.pcap","avl/delay_10_oracle_0.pcap"]}` + + we'll use the following command: `get_last_value("template_info.data[2]”)`. + + This will produce the following result: + + *`avl/delay_10_https_0.pcap`* + + + +== Usage Examples + +=== Exmaple #1: Checking T-Rex status and Launching T-Rex + +The following program checks T-Rex status, and later on launches it, querying its status along different time slots. + +[source, python] +---- +import time + +trex = CTRexClient('trex-name') +print "Before Running, T-Rex status is: ", trex.is_running() # <1> +print "Before Running, T-Rex status is: ", trex.get_running_status() # <2> + +ret = trex.start_trex( c = 2, # <3> + m = 0.1, + d = 40, + f = 'avl/sfr_delay_10_1g.yaml', + nc = True, + p = True, + l = 1000) + +print "After Starting, T-Rex status is: ", trex.is_running(), trex.get_running_status() + +time.sleep(10) # <4> + +print "Is T-Rex running? ", trex.is_running(), trex.get_running_status() # <5> +---- + +<1> `is_running()` returns a boolean and checks if T-Rex is running or not. + +<2> `get_running_status()` returns a Python dictionary with T-Rex state, along with a verbose field containing extra info, if available. + +<3> T-Rex lanching. All types of inputs are supported. Some fields (such as 'f' and 'd' are mandatory). + +<4> Going to sleep for few seconds, allowing T-Rex to start. + +<5> Checking out with T-Rex status again, printing both a boolean return value and a full status. + +This code will prompt the following output, assuming a server was launched on the T-Rex machine. + +---- +Connecting to T-Rex @ http://trex-dan:8090/ ... +Before Running, T-Rex status is: False +Before Running, T-Rex status is: {u'state': <TRexStatus.Idle: 1>, u'verbose': u'T-Rex is Idle'} + <1> <1> + +After Starting, T-Rex status is: False {u'state': <TRexStatus.Starting: 2>, u'verbose': u'T-Rex is starting'} + <1> <1> +Is T-Rex running? True {u'state': <TRexStatus.Running: 3>, u'verbose': u'T-Rex is Running'} + <1> <1> +---- + +<1> When looking at T-Rex status, both an enum status (`Idle, Starting, Running`) and verbose output are available. + + +=== Exmaple #2: Checking T-Rex status and Launching T-Rex with 'BAD PARAMETERS' + +The following program checks T-Rex status, and later on launches it with wrong input ('mdf' is not legal option), hence T-Rex run will not start and a message will be available. + +[source, python] +---- +import time + +trex = CTRexClient('trex-name') +print "Before Running, T-Rex status is: ", trex.is_running() # <1> +print "Before Running, T-Rex status is: ", trex.get_running_status() # <2> + +ret = trex.start_trex( c = 2, # <3> +#<4> mdf = 0.1, + d = 40, + f = 'avl/sfr_delay_10_1g.yaml', + nc = True, + p = True, + l = 1000) + +print "After Starting, T-Rex status is: ", trex.is_running(), trex.get_running_status() + +time.sleep(10) # <5> + +print "Is T-Rex running? ", trex.is_running(), trex.get_running_status() # <6> +---- + +<1> `is_running()` returns a boolean and checks if T-Rex is running or not. + +<2> `get_running_status()` returns a Python dictionary with T-Rex state, along with a verbose field containing extra info, if available. + +<3> T-Rex lanching. All types of inputs are supported. Some fields (such as 'f' and 'c' are mandatory). + +<4> Wrong parameter ('mdf') injected. + +<5> Going to sleep for few seconds, allowing T-Rex to start. + +<6> Checking out with T-Rex status again, printing both a boolean return value and a full status. + +This code will prompt the following output, assuming a server was launched on the T-Rex machine. +---- +Connecting to T-Rex @ http://trex-dan:8090/ ... +Before Running, T-Rex status is: False +Before Running, T-Rex status is: {u'state': <TRexStatus.Idle: 1>, u'verbose': u'T-Rex is Idle'} + <1> <1> + +After Starting, T-Rex status is: False {u'state': <TRexStatus.Starting: 2>, u'verbose': u'T-Rex is starting'} + <1> <1> +Is T-Rex running? False {u'state': <TRexStatus.Idle: 1>, u'verbose': u'T-Rex run failed due to wrong input parameters, or due to reachability issues.'} + <2> <2> +---- + +<1> When looking at T-Rex status, both an enum status (`Idle, Starting, Running`) and verbose output are available. + +<2> After T-Rex lanuching failed, a message indicating the failure reason. However, T-Rex is back Idle, ready to handle another launching request. + + +=== Exmaple #3: Launching T-Rex, let it run until custom condition is satisfied + +The following program will launch T-Rex, and poll its result data until custom condition function returns `True`. + In this case, the condition function is simply named `condition`. + +Once the condition is met, T-Rex run will be terminated. + +[source, python] +---- +print "Before Running, T-Rex status is: ", trex.get_running_status() + + print "Starting T-Rex..." + ret = trex.start_trex( c = 2, + mdf = 0.1, + d = 1000, + f = 'avl/sfr_delay_10_1g.yaml', + nc = True, + p = True, + l = 1000) + + def condition (result_obj): #<1> + return result_obj.get_current_tx_rate()['m_tx_pps'] > 200000 + + res = trex.sample_until_condition(condition) #<2> + + print res #<3> + val_list = res.get_value_list("trex-global.data", "m_tx_expected_\w+") #<4> +---- + +<1> The `condition` function defines when to stop T-Rex. In this case, when T-Rex's current tx (in pps) exceeds 200000. + +<2> The condition is passed to `sample_until_condition` method, which will block until either the condition is met or an 'Exception' is raised. + +<3> Once satisfied, `res` variable holds the first result object on which the condition satisfied. At this point, T-Rex status is 'Idle' and another run can be requested from the server. + +<4> Further custom processing can be made on the result object, regardless of other T-Rex runs. + +<<< + +=== Exmaple #4: Launching T-Rex, monitor live data and stopping on demand + +The following program will launch T-Rex, and while it runs poll the server (every 5 seconds) for running inforamtion, such as latency, drops, and other extractable parameters. + +Then, after some criteria was met, T-Rex execution is terminated, enabeling others to use the resource instead of waiting for the entire execution to finish. + +[source, python] +---- +print "Before Running, T-Rex status is: ", trex.get_running_status() + + print "Starting T-Rex..." + ret = trex.start_trex( c = 2, + mdf = 0.1, + d = 100, + f = 'avl/sfr_delay_10_1g.yaml', + nc = True, + p = True, + l = 1000) + + last_res = dict() + while trex.is_running(dump_out = last_res): #<1> + print '\n\n*****************************************' + print "RECEIVED DUMP:" + print last_res, "\n\n\n" + + print "CURRENT RESULT OBJECT" + obj = trex.get_result_obj() + #<2> # Custom data processing is done here, for example: + print obj.get_value_list("trex-global.data.m_tx_bps") + time.sleep(5) #<3> + + print "Terminating T-Rex..." + ret = trex.stop_trex() #<4> +---- + +<1> Iterate as long as T-Rex is running. + + In this case the latest dump is also saved into `last_res` variable, so easier access for that data is available, although not needed most of the time. + +<2> Data processing. This is fully customizable for the relevant test initiated. + +<3> The sampling rate is flexibale and can be configured depending on the desired output. + +<4> T-Rex termination. + +<<< + +=== Exmaple #5: Launching T-Rex, let it run until finished + +The following program will launch T-Rex, and poll it automatically until run finishes. The polling rate is customisable (in this case, every 10 seconds) using `time_between_samples` argument. + +[source, python] +---- +print "Before Running, T-Rex status is: ", trex.get_running_status() + + print "Starting T-Rex..." + ret = trex.start_trex( c = 2, #<1> + mdf = 0.1, + d = 1000, + f = 'avl/sfr_delay_10_1g.yaml', + nc = True, + p = True, + l = 1000) + + res = trex.sample_to_run_finish(time_between_samples = 10) #<2> + + print res #<3> + val_list = res.get_value_list("trex-global.data", "m_tx_expected_\w+") #<4> +---- + +<1> T-Rex run initialization. + +<2> Define the sample rate and block until T-Rex run ends. Once this method returns (assuming no error), T-Rex result object will contain the samples collected allong T-Rex run, limited to the history size footnoteref:[For example, For example for history sized 100 only the latest 100 samples will be available despite sampling more than that during T-Rex run.]. + +<3> Once finished, `res` variable holds the latest result object. + +<4> Further custom processing can be made on the result object, regardless of other T-Rex runs.
\ No newline at end of file diff --git a/trex_control_plane_peek.asciidoc b/trex_control_plane_peek.asciidoc new file mode 100755 index 00000000..530da965 --- /dev/null +++ b/trex_control_plane_peek.asciidoc @@ -0,0 +1,225 @@ +T-Rex Control Plane Design - Phase 1 peek
+=========================================
+:author: Dan Klein
+:email: <danklei@cisco.com>
+:revnumber: 1.0
+:quotes.++:
+:numbered:
+
+
+
+=== T-Rex traffic generator
+
+T-Rex traffic generator is a tool design the benchmark platforms with realistic traffic.
+This is a work-in-progress product, which is under constant developement, new features are added and support for more router's fuctionality is achieved.
+
+=== T-Rex Control
+
+T-Rex control plane is under developement, and a phase 1 is planned to be published soon (Apr 2015). +
+This document will shortly describe the planned control plane for T-Rex, which is planned to be more scalable and support automation more intuitively.
+
+==== T-Rex Control Plane - High Level
+
+T-Rex control plane is based on a JSON RPC transactions between clients and server. +
+Each T-Rex machine will have a server running on it, closely interacting with T-Rex (clients do not approach T-Rex directly). +
+As future feature, and as multiple T-Rexes might run on the same machine, single server shall serve all T-Rexes running a machine.
+
+The client is a Python based application that created `TRexClient` instances. +
+Using class methods, the client interacts with T-Rex server, and enable it to perform the following commands:
+
+ 1. Start T-Rex run (custom parameters supported).
+
+ 2. Stop T-Rex run.
+
+ 3. Check what is the T-Rex status (possible states: `Idle, Starting, Running`).
+
+ 4. Poll (by customize sampling) the server and get live results from T-Rex **while still running**.
+
+ 5. Get custom T-Rex stats based on a window of saved history of latest 'N' polling results.
+
+
+==== T-Rex Control Plane - Example crumbs
+
+
+
+ - **Exmaple #1: Checking T-Rex status and Launching T-Rex**
+ The following program checks T-Rex status, and later on launches it, querying its status along different time slots.
+
+[source, python]
+----
+import time
+
+trex = CTRexClient('trex-name')
+print "Before Running, T-Rex status is: ", trex.is_running() # <1>
+print "Before Running, T-Rex status is: ", trex.get_running_status() # <2>
+
+ret = trex.start_trex( c = 2, # <3>
+ m = 0.1,
+ d = 20,
+ f = 'avl/sfr_delay_10_1g.yaml',
+ nc = True,
+ p = True,
+ l = 1000)
+
+print "After Starting, T-Rex status is: ", trex.is_running(), trex.get_running_status()
+
+time.sleep(10) # <4>
+
+print "Is T-Rex running? ", trex.is_running(), trex.get_running_status() # <5>
+----
+
+<1> `is_running()` returns a boolean and checks if T-Rex is running or not.
+
+<2> `get_running_status()` returns a Python dictionary with T-Rex state, along with a verbose field containing extra info, if available.
+
+<3> T-Rex lanching. All types of inputs are supported. Some fields (such as 'f' and 'c' are mandatory).
+
+<4> Going to sleep for few seconds, allowing T-Rex to start.
+
+<5> Checking out with T-Rex status again, printing both a boolean return value and a full status.
+
+This code will prompt the following output, assuming a server was launched on the T-Rex machine.
+
+----
+Connecting to T-Rex @ http://trex-dan:8090/ ...
+Before Running, T-Rex status is: False
+Before Running, T-Rex status is: {u'state': <TRexStatus.Idle: 1>, u'verbose': u'T-Rex is Idle'}
+ <1> <1>
+
+After Starting, T-Rex status is: False {u'state': <TRexStatus.Starting: 2>, u'verbose': u'T-Rex is starting'}
+ <1> <1>
+Is T-Rex running? True {u'state': <TRexStatus.Running: 3>, u'verbose': u'T-Rex is Running'}
+ <1> <1>
+----
+
+<1> When looking at T-Rex status, both an enum status (`Idle, Starting, Running`) and verbose output are available.
+
+
+ * **Exmaple #2: Checking T-Rex status and Launching T-Rex with 'BAD PARAMETERS'**
+ The following program checks T-Rex status, and later on launches it with wrong input ('mdf' is not legal option), hence T-Rex run will not start and a message will be available.
+
+[source, python]
+----
+import time
+
+trex = CTRexClient('trex-name')
+print "Before Running, T-Rex status is: ", trex.is_running() # <1>
+print "Before Running, T-Rex status is: ", trex.get_running_status() # <2>
+
+ret = trex.start_trex( c = 2, # <3>
+#<4> mdf = 0.1,
+ d = 20,
+ f = 'avl/sfr_delay_10_1g.yaml',
+ nc = True,
+ p = True,
+ l = 1000)
+
+print "After Starting, T-Rex status is: ", trex.is_running(), trex.get_running_status()
+
+time.sleep(10) # <5>
+
+print "Is T-Rex running? ", trex.is_running(), trex.get_running_status() # <6>
+----
+
+<1> `is_running()` returns a boolean and checks if T-Rex is running or not.
+
+<2> `get_running_status()` returns a Python dictionary with T-Rex state, along with a verbose field containing extra info, if available.
+
+<3> T-Rex lanching. All types of inputs are supported. Some fields (such as 'f' and 'c' are mandatory).
+
+<4> Wrong parameter ('mdf') injected.
+
+<5> Going to sleep for few seconds, allowing T-Rex to start.
+
+<6> Checking out with T-Rex status again, printing both a boolean return value and a full status.
+
+This code will prompt the following output, assuming a server was launched on the T-Rex machine.
+----
+Connecting to T-Rex @ http://trex-dan:8090/ ...
+Before Running, T-Rex status is: False
+Before Running, T-Rex status is: {u'state': <TRexStatus.Idle: 1>, u'verbose': u'T-Rex is Idle'}
+ <1> <1>
+
+After Starting, T-Rex status is: False {u'state': <TRexStatus.Starting: 2>, u'verbose': u'T-Rex is starting'}
+ <1> <1>
+Is T-Rex running? False {u'state': <TRexStatus.Idle: 1>, u'verbose': u'T-Rex run failed due to wrong input parameters, or due to reachability issues.'}
+ <2> <2>
+----
+
+<1> When looking at T-Rex status, both an enum status (`Idle, Starting, Running`) and verbose output are available.
+
+<2> After T-Rex lanuching failed, a message indicating the failure reason. However, T-Rex is back Idle, ready to handle another launching request.
+
+
+ * **Exmaple #3: Launching T-Rex, monitor live data and stopping on demand**
+ The following program will launch T-Rex, and while it runs poll the server (every 5 seconds) for running inforamtion, such as latency, drops, and other extractable parameters. +
+ Then, after some criteria was met, T-Rex execution is terminated, enabeling others to use the resource instead of waiting for the entire execution to finish.
+
+[source, python]
+----
+print "Before Running, T-Rex status is: ", trex.get_running_status()
+
+ print "Starting T-Rex..."
+ ret = trex.start_trex( c = 2,
+ mdf = 0.1,
+ d = 100,
+ f = 'avl/sfr_delay_10_1g.yaml',
+ nc = True,
+ p = True,
+ l = 1000)
+
+ print "After Starting, T-Rex status is: ", trex.is_running(), trex.get_running_status()
+ print "sleeping 20 secs.."
+ time.sleep(20)
+ for i in range(5):
+ print "Is T-Rex running? ", trex.is_running(), trex.get_running_status() #<1>
+ #<2> received_info = trex.get_running_info()
+ #<3> # Custom data processing is done here
+ #<4> time.sleep(5)
+
+ print "Terminating T-Rex..."
+ #<5> ret = trex.stop_trex()
+ print "After stopping, T-Rex status is: ", trex.is_running(), trex.get_running_status() #<6>
+----
+
+<1> Running queries is still optional, although not mandatory in order to get stats.
+
+<2> `get_running_info()` will return the latest data dump available from T-Rex. +
+ Some aditional data manipulation and queries are under developement, including manipulation over number of dumps, which is useful for avoiding "spikes" of singular behavior.
+
+<3> Data processing. This is fully customizable for the relevant test initiated.
+
+<4> The sampling rate is flexibale and can be configured depending on the desired output.
+
+<5> T-Rex termination.
+
+<6> Post-termination check for status.
+
+
+This code will prompt the following output, assuming a server was launched on the T-Rex machine.
+----
+Connecting to T-Rex @ http://trex-dan:8090/ ...
+Before Running, T-Rex status is: False
+Before Running, T-Rex status is: {u'state': <TRexStatus.Idle: 1>, u'verbose': u'T-Rex is Idle'}
+Starting T-Rex...
+After Starting, T-Rex status is: False {u'state': <TRexStatus.Starting: 2>, u'verbose': u'T-Rex is starting'}
+
+<1> Is T-Rex running? True {u'state': <TRexStatus.Running: 3>, u'verbose': u'T-Rex is Running'}
+
+<1> Is T-Rex running? True {u'state': <TRexStatus.Running: 3>, u'verbose': u'T-Rex is Running'}
+
+<1> Is T-Rex running? True {u'state': <TRexStatus.Running: 3>, u'verbose': u'T-Rex is Running'}
+
+<1> Is T-Rex running? True {u'state': <TRexStatus.Running: 3>, u'verbose': u'T-Rex is Running'}
+
+<1> Is T-Rex running? True {u'state': <TRexStatus.Running: 3>, u'verbose': u'T-Rex is Running'}
+
+Before terminating, T-Rex status is: True {u'state': <TRexStatus.Running: 3>, u'verbose': u'T-Rex is Running'}
+Terminating T-Rex...
+#<2> After stopping, T-Rex status is: False {u'state': <TRexStatus.Idle: 1>, u'verbose': u'T-Rex finished (terminated).'}
+
+----
+
+<1> Polling T-Rex status while in a data polling loop.
+
+<2> After termination, we can see that T-Rex is back idle, also the `verbose` field shows the stop reason
\ No newline at end of file diff --git a/trex_preso.asciidoc b/trex_preso.asciidoc new file mode 100755 index 00000000..50f4ffc7 --- /dev/null +++ b/trex_preso.asciidoc @@ -0,0 +1,1312 @@ +TRex realistic traffic generator +================================ +:author: hhaim +:email: <hhaim@cisco.com> +:description: TRex Low cost, High scale, realistic traffic generator +:revdate: 2014-11-01 +:revnumber: 0.2 +:deckjs_theme: swiss +:deckjs_transition: horizontal-slide +:scrollable: +:web_server_url: http://trex-tgn.cisco.com/trex + + + +== What problem is being solved? + +* Network elements include complex statful features +* Require testing with statful and real traffic mix +* Traffic generators of statful/realistic traffic are: +** Expensive ~$100-500K +** Not scalable for high rates +** Not flexible +* Due to cost, quality is impacted +** Limited access / testing +** Late testing +** No standard + +++++++++++++++++++ +<script type="text/javascript" + src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"> +</script> + +<script src="http://d3js.org/d3.v3.min.js" charset="utf-8"></script> + +<script src="my_chart.js"></script> + +<style> +.axis path, +.axis line { + fill: none; + stroke: #000; + shape-rendering: crispEdges; +} + +.dot { + stroke: #000; +} +</style> + + +<style type="text/css"> + +h1 { + font-size: 2.5em; +} + +h2 { + font-size: 1.5em; + color: #CD7300; + border-bottom-color: #000; +} + +h7 { + font-size: 4.5em; + color: #CD7300; + position: relative; + top: auto; + text-align: center; + padding: 0; + -webkit-transform: none; + -moz-transform: none; + -ms-transform: none; + -o-transform: none; + transform: none; + padding: 0 48px; + position: absolute; + left: 0; + right: 0; + top: 50%; +} + +h8 { + font-size: 2.25em; + font-weight: bold; + padding-top: .5em; + margin: 0 0 .66666em 0; + border-top: 3px solid #888; + color: #c00; + border-top-color: #ccc; + left: 0; + right: 0; + top: 40%; +} + + +html, body { + height: 100%; + margin: 0 auto; + max-width: 1000px; +} + +</style> + +<script> + $('#title-slide').css("background-image", "url('images/trex_logo.png')"); + $('#title-slide').css("background-repeat","no-repeat"); + $('#title-slide').css("background-position","center"); + $('h1').html(''); + $('h3').html('<font size="4">Hanoch Haim v1.2</font>'); + $('h4').html('<font size="4">04/2015</font>'); + </script> + +++++++++++++++++++ + + +== What is TRex? + + +* TRex is a *statefull* *traffic* *generator* tool based on a smart replay of real flows (not a full TCP/IP stack,yet) +++++++++++++++++++ +<img src="images/ucs200_2.png" alt="title=" align="right" > +++++++++++++++++++ +* Generates, manipulates and amplifies based on templates of a real/captured flows. +* Templates are processed offline +* *High* *performance* full line rate +** tx= up to 200Gb/sec rx=up to 200Gb/sec +* *Low* *cost* C220M UCS-1RU, Cisco internal eqip +* Generate both sides of the traffic Clients and Servers +* *Standard* *hardware* (X86/Intel NIC I350,82599,XL710) +++++++++++++++++++ +<img src="images/Intel520.png" alt="title=" align="right" > +++++++++++++++++++ +* Flexible and Open Software (DPDK) +* Support Virtualization +* Virtual interface support *E1000* , *VMXNET3*. Enabler for +** Amazon AWS +** Cisco LaaS + +== Realistic traffic model + +++++++++++++++++++ +<img src="images/trex_model.png" alt="title=" align="center" > +++++++++++++++++++ + +== TRex high level software architecture + +++++++++++++++++++ +<img src="images/trex_desing.png" alt="title=" align="right" > +++++++++++++++++++ + +* *DPDK* +** user space driver +** Big TLB +** Utilize all DDR banks + +* *Multi-Threaded* +** Scale linearly +** ~20Gb/sec per DP core + +* *No TCP-IP* +** Fast events scheduler +** Generate flows template +** Can support 1K templates + +* *Slow-path flexibility (per-flow)* +** Client/server generation +** Measure latency +** Measure flow order +** NAT translation learning + +== TRex basic flows generation algorithm + +++++++++++++++++++ +<img src="images/trex_algo.png" alt="title=" align="center" > +++++++++++++++++++ + +== DNS simple profile example + +* traffic profile is in YAML fomat + +[source,python] +---- +$more cap2/dns_test.yaml +- duration : 10.0 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.255" + servers_start : "48.0.0.1" + servers_end : "48.0.0.255" + dual_port_mask : "1.0.0.0" + tcp_aging : 1 + udp_aging : 1 + cap_info : + - name: cap2/dns.pcap <1> + cps : 1.0 <2> +---- +<1> the pcap file that include DNS cap file that will be replicate +<2> how many connection per second to generate, 1.0 means 1 connection per secod + +image:images/dns_wireshark.png[title="generator"] + +== DNS simple profile output + +.Formated results +[format="csv",cols="1^,2^,1^,1^,2^,1^,2^,1^", options="header"] +|================= + pkt,time sec,fid,flow-pkt-id,client_ip,client_port,server_ip ,direction + 1 , 0.010000 , 1 , 1 , 16.0.0.1 , 1024 , 48.0.0.1 , -> + 2 , 0.020000 , 1 , 2 , 16.0.0.1 , 1024 , 48.0.0.1 , <- + 3 , 2.010000 , 2 , 1 , 16.0.0.2 , 1024 , 48.0.0.2 , -> + 4 , 2.020000 , 2 , 2 , 16.0.0.2 , 1024 , 48.0.0.2 , <- + 5 , 3.010000 , 3 , 1 , 16.0.0.3 , 1024 , 48.0.0.3 , -> + 6 , 3.020000 , 3 , 2 , 16.0.0.3 , 1024 , 48.0.0.3 , <- + 7 , 4.010000 , 4 , 1 , 16.0.0.4 , 1024 , 48.0.0.4 , -> + 8 , 4.020000 , 4 , 2 , 16.0.0.4 , 1024 , 48.0.0.4 , <- + 9 , 5.010000 , 5 , 1 , 16.0.0.5 , 1024 , 48.0.0.5 , -> + 10 , 5.020000 , 5 , 2 , 16.0.0.5 , 1024 , 48.0.0.5 , <- + 11 , 6.010000 , 6 , 1 , 16.0.0.6 , 1024 , 48.0.0.6 , -> +|================= + +== DNS simple profile chart + + ++++++++++++++++++++++++++++++++++ +<div id="chart1" style="font : 10px sans-serif"; ></div> + +<script> +var dns_data=[ +[ 0.010000 , 1 , 0 , 1 ], +[ 0.060000 , 1 , 0 , 2 ], +[ 0.210000 , 2 , 0 , 1 ], +[ 0.260000 , 2 , 0 , 2 ], +[ 0.310000 , 3 , 0 , 1 ], +[ 0.360000 , 3 , 0 , 2 ], +[ 0.410000 , 4 , 0 , 1 ], +[ 0.460000 , 4 , 0 , 2 ], +[ 0.510000 , 5 , 0 , 1 ], +[ 0.560000 , 5 , 0 , 2 ], +[ 0.610000 , 6 , 0 , 1 ], +[ 0.660000 , 6 , 0 , 2 ], +[ 0.710000 , 7 , 0 , 1 ], +[ 0.760000 , 7 , 0 , 2 ], +[ 0.810000 , 8 , 0 , 1 ], +[ 0.860000 , 8 , 0 , 2 ], +[ 0.910000 , 9 , 0 , 1 ], +[ 0.960000 , 9 , 0 , 2 ], +[ 1.010000 , 10 , 0 , 1 ], +[ 1.060000 , 10 , 0 , 2 ], +]; +chart("#chart1",dns_data,["dns"],"time-sec","flow-id"); +</script> + ++++++++++++++++++++++++++++++++++ + +== HTTP & DNS profile example + +[source,python] +---- +- duration : 1.0 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.0.10" + servers_start : "48.0.0.1" + servers_end : "48.0.0.3" + dual_port_mask : "1.0.0.0" + tcp_aging : 1 + udp_aging : 1 + cap_ipg : true + cap_info : + - name: cap2/dns.pcap + cps : 10.0 <1> + - name: avl/delay_10_http_browsing_0.pcap + cps : 2.0 <1> +---- +<1> Diffrent CPS + +* Inter packet Gap (IPG) is taken from pcap file +* IPG can be manualy set + +== HTTP & DNS profile chart + ++++++++++++++++++++++++++++++++++ +<div id="chart3" style="font : 10px sans-serif"; ></div> + +<script> +var chart3_data=[ +[ 0.010000 , 1 , 0 , 1 ], +[ 0.030944 , 1 , 0 , 2 ], +[ 0.093333 , 2 , 1 , 1 ], +[ 0.104362 , 2 , 1 , 2 ], +[ 0.115385 , 2 , 1 , 3 ], +[ 0.115394 , 2 , 1 , 4 ], +[ 0.126471 , 2 , 1 , 5 ], +[ 0.126484 , 2 , 1 , 6 ], +[ 0.137530 , 2 , 1 , 7 ], +[ 0.148609 , 2 , 1 , 8 ], +[ 0.148621 , 2 , 1 , 9 ], +[ 0.148635 , 2 , 1 , 10 ], +[ 0.159663 , 2 , 1 , 11 ], +[ 0.170750 , 2 , 1 , 12 ], +[ 0.170762 , 2 , 1 , 13 ], +[ 0.170774 , 2 , 1 , 14 ], +[ 0.176667 , 3 , 0 , 1 ], +[ 0.181805 , 2 , 1 , 15 ], +[ 0.181815 , 2 , 1 , 16 ], +[ 0.192889 , 2 , 1 , 17 ], +[ 0.192902 , 2 , 1 , 18 ], +[ 0.192914 , 2 , 1 , 19 ], +[ 0.192927 , 2 , 1 , 20 ], +[ 0.192939 , 2 , 1 , 21 ], +[ 0.192951 , 2 , 1 , 22 ], +[ 0.197611 , 3 , 0 , 2 ], +[ 0.203944 , 2 , 1 , 23 ], +[ 0.203950 , 2 , 1 , 24 ], +[ 0.203956 , 2 , 1 , 25 ], +[ 0.214620 , 2 , 1 , 26 ], +[ 0.214633 , 2 , 1 , 27 ], +[ 0.214645 , 2 , 1 , 28 ], +[ 0.214658 , 2 , 1 , 29 ], +[ 0.214671 , 2 , 1 , 30 ], +[ 0.214682 , 2 , 1 , 31 ], +[ 0.214695 , 2 , 1 , 32 ], +[ 0.214707 , 2 , 1 , 33 ], +[ 0.225264 , 2 , 1 , 34 ], +[ 0.225269 , 2 , 1 , 35 ], +[ 0.225274 , 2 , 1 , 36 ], +[ 0.225279 , 2 , 1 , 37 ], +[ 0.260000 , 4 , 0 , 1 ], +[ 0.280944 , 4 , 0 , 2 ], +[ 0.343333 , 5 , 0 , 1 ], +[ 0.364277 , 5 , 0 , 2 ], +[ 0.426667 , 6 , 0 , 1 ], +[ 0.447611 , 6 , 0 , 2 ], +[ 0.593333 , 7 , 0 , 1 ], +[ 0.614277 , 7 , 0 , 2 ], +[ 0.676667 , 8 , 1 , 1 ], +[ 0.687696 , 8 , 1 , 2 ], +[ 0.698719 , 8 , 1 , 3 ], +[ 0.698728 , 8 , 1 , 4 ], +[ 0.709805 , 8 , 1 , 5 ], +[ 0.709818 , 8 , 1 , 6 ], +[ 0.720864 , 8 , 1 , 7 ], +[ 0.731943 , 8 , 1 , 8 ], +[ 0.731955 , 8 , 1 , 9 ], +[ 0.731968 , 8 , 1 , 10 ], +[ 0.742997 , 8 , 1 , 11 ], +[ 0.754084 , 8 , 1 , 12 ], +[ 0.754096 , 8 , 1 , 13 ], +[ 0.754108 , 8 , 1 , 14 ], +[ 0.760000 , 9 , 0 , 1 ], +[ 0.765139 , 8 , 1 , 15 ], +[ 0.765148 , 8 , 1 , 16 ], +[ 0.776223 , 8 , 1 , 17 ], +[ 0.776236 , 8 , 1 , 18 ], +[ 0.776248 , 8 , 1 , 19 ], +[ 0.776261 , 8 , 1 , 20 ], +[ 0.776273 , 8 , 1 , 21 ], +[ 0.776285 , 8 , 1 , 22 ], +[ 0.780944 , 9 , 0 , 2 ], +[ 0.787278 , 8 , 1 , 23 ], +[ 0.787284 , 8 , 1 , 24 ], +[ 0.787289 , 8 , 1 , 25 ], +[ 0.797954 , 8 , 1 , 26 ], +[ 0.797967 , 8 , 1 , 27 ], +[ 0.797979 , 8 , 1 , 28 ], +[ 0.797992 , 8 , 1 , 29 ], +[ 0.798004 , 8 , 1 , 30 ], +[ 0.798016 , 8 , 1 , 31 ], +[ 0.798029 , 8 , 1 , 32 ], +[ 0.798041 , 8 , 1 , 33 ], +[ 0.808598 , 8 , 1 , 34 ], +[ 0.808603 , 8 , 1 , 35 ], +[ 0.808608 , 8 , 1 , 36 ], +[ 0.808613 , 8 , 1 , 37 ], +[ 0.843333 , 10 , 0 , 1 ], +[ 0.864277 , 10 , 0 , 2 ], +[ 0.926667 , 11 , 0 , 1 ], +[ 0.947611 , 11 , 0 , 2 ], +[ 1.010000 , 12 , 0 , 1 ], +[ 1.030944 , 12 , 0 , 2 ], +]; +chart("#chart3",chart3_data,["dns","http"],"time-sec","flow-id"); +</script> ++++++++++++++++++++++++++++++++++ + +== EMIX traffic profile + +++++++++++++++++++ +<img src="images/trex_sfr_profile.png" alt="title=" align="right" > +++++++++++++++++++ + +* EMIX traffic profile suggested. used for NBAR/AVC tests +* Property +** BW: 70% TCP (http, mail) , 30% UDP (rtp) +** Connections: 50% TCP (http, mail) , 50% UDP (rtp, dns) +** Avg. Packet Size (~585B) +** PPS= 221K per 1Gb/sec +** CPS = 4K per 1Gb/sec +** Flows per 1Gb/sec = 50K +** Avg. Packets per flow (50) + +== EMIX YAML profile + +.Simplified version of EMIX YAML profile +[source,python] +---- +- duration : 0.1 + generator : + distribution : "seq" + clients_start : "16.0.0.1" + clients_end : "16.0.1.255" + servers_start : "48.0.0.1" + servers_end : "48.0.20.255" + dual_port_mask : "1.0.0.0" + cap_ipg : true + cap_info : + - name: avl/delay_10_http_get_0.pcap + cps : 404.52 + - name: avl/delay_10_http_post_0.pcap + cps : 404.52 + - name: avl/delay_10_https_0.pcap + cps : 130.8745 + - name: avl/delay_10_http_browsing_0.pcap + cps : 709.89 + - name: avl/delay_10_exchange_0.pcap + cps : 253.81 + - name: avl/delay_10_mail_pop_2.pcap + cps : 4.759 + - name: avl/delay_10_oracle_0.pcap + cps : 79.3178 + - name: avl/delay_10_rtp_160k_full.pcap + cps : 2.776 + - name: avl/delay_10_smtp_0.pcap + cps : 7.3369 + - name: avl/delay_10_sip_video_call_full.pcap + cps : 29.347 + - name: avl/delay_10_citrix_0.pcap + cps : 43.6248 + - name: avl/delay_10_dns_0.pcap + cps : 1975.015 +---- + +== EMIX profile chart + ++++++++++++++++++++++++++++++++++ +<div id="chart4" style="font : 10px sans-serif"; ></div> + +<script> + +var sfr_data=[ +[ 0.010000 , 1 , 0 , 1 ], +[ 0.010245 , 2 , 1 , 1 ], +[ 0.010490 , 3 , 2 , 1 ], +[ 0.010735 , 4 , 3 , 1 ], +[ 0.010979 , 5 , 4 , 1 ], +[ 0.011224 , 6 , 5 , 1 ], +[ 0.011469 , 7 , 6 , 1 ], +[ 0.011714 , 8 , 7 , 1 ], +[ 0.011959 , 9 , 8 , 1 ], +[ 0.012204 , 10 , 9 , 1 ], +[ 0.012449 , 11 , 10 , 1 ], +[ 0.012694 , 12 , 11 , 1 ], +[ 0.012938 , 13 , 12 , 1 ], +[ 0.013183 , 14 , 13 , 1 ], +[ 0.013428 , 15 , 14 , 1 ], +[ 0.013673 , 16 , 15 , 1 ], +[ 0.013918 , 17 , 16 , 1 ], +[ 0.014163 , 18 , 17 , 1 ], +[ 0.014408 , 19 , 0 , 1 ], +[ 0.014652 , 20 , 1 , 1 ], +[ 0.014897 , 21 , 3 , 1 ], +[ 0.015142 , 22 , 4 , 1 ], +[ 0.015387 , 23 , 17 , 1 ], +[ 0.015632 , 24 , 0 , 1 ], +[ 0.015877 , 25 , 1 , 1 ], +[ 0.016122 , 26 , 3 , 1 ], +[ 0.016367 , 27 , 17 , 1 ], +[ 0.016611 , 28 , 3 , 1 ], +[ 0.016856 , 29 , 17 , 1 ], +[ 0.017101 , 30 , 3 , 1 ], +[ 0.017346 , 31 , 17 , 1 ], +[ 0.017591 , 32 , 0 , 1 ], +[ 0.017836 , 33 , 1 , 1 ], +[ 0.018081 , 34 , 3 , 1 ], +[ 0.018325 , 35 , 17 , 1 ], +[ 0.018456 , 15 , 14 , 2 ], +[ 0.018570 , 36 , 2 , 1 ], +[ 0.018815 , 37 , 17 , 1 ], +[ 0.019060 , 38 , 4 , 1 ], +[ 0.019305 , 39 , 17 , 1 ], +[ 0.019550 , 40 , 3 , 1 ], +[ 0.019795 , 41 , 17 , 1 ], +[ 0.021137 , 1 , 0 , 2 ], +[ 0.021294 , 2 , 1 , 2 ], +[ 0.021534 , 3 , 2 , 2 ], +[ 0.021764 , 4 , 3 , 2 ], +[ 0.022007 , 5 , 4 , 2 ], +[ 0.022251 , 6 , 5 , 2 ], +[ 0.022505 , 7 , 6 , 2 ], +[ 0.022768 , 8 , 7 , 2 ], +[ 0.022989 , 9 , 8 , 2 ], +[ 0.023245 , 10 , 9 , 2 ], +[ 0.023490 , 11 , 10 , 2 ], +[ 0.023747 , 12 , 11 , 2 ], +[ 0.023963 , 13 , 12 , 2 ], +[ 0.024212 , 14 , 13 , 2 ], +[ 0.024972 , 17 , 16 , 2 ], +[ 0.025262 , 18 , 17 , 2 ], +[ 0.025545 , 19 , 0 , 2 ], +[ 0.025701 , 20 , 1 , 2 ], +[ 0.025926 , 21 , 3 , 2 ], +[ 0.026169 , 22 , 4 , 2 ], +[ 0.026486 , 23 , 17 , 2 ], +[ 0.026769 , 24 , 0 , 2 ], +[ 0.026926 , 25 , 1 , 2 ], +[ 0.027151 , 26 , 3 , 2 ], +[ 0.027465 , 27 , 17 , 2 ], +[ 0.027640 , 28 , 3 , 2 ], +[ 0.027955 , 29 , 17 , 2 ], +[ 0.028130 , 30 , 3 , 2 ], +[ 0.028445 , 31 , 17 , 2 ], +[ 0.028728 , 32 , 0 , 2 ], +[ 0.028885 , 33 , 1 , 2 ], +[ 0.029110 , 34 , 3 , 2 ], +[ 0.029424 , 35 , 17 , 2 ], +[ 0.029614 , 36 , 2 , 2 ], +[ 0.029914 , 37 , 17 , 2 ], +[ 0.030087 , 38 , 4 , 2 ], +[ 0.030404 , 39 , 17 , 2 ], +[ 0.030579 , 40 , 3 , 2 ], +[ 0.030894 , 41 , 17 , 2 ], +[ 0.032188 , 1 , 0 , 3 ], +[ 0.032197 , 1 , 0 , 4 ], +[ 0.032341 , 2 , 1 , 3 ], +[ 0.032367 , 2 , 1 , 4 ], +[ 0.032379 , 2 , 1 , 5 ], +[ 0.032576 , 3 , 2 , 3 ], +[ 0.032583 , 3 , 2 , 4 ], +[ 0.032787 , 4 , 3 , 3 ], +[ 0.032796 , 4 , 3 , 4 ], +[ 0.032931 , 16 , 15 , 2 ], +[ 0.033031 , 5 , 4 , 3 ], +[ 0.033052 , 5 , 4 , 4 ], +[ 0.033065 , 5 , 4 , 5 ], +[ 0.033272 , 6 , 5 , 3 ], +[ 0.033460 , 15 , 14 , 3 ], +[ 0.033527 , 7 , 6 , 3 ], +[ 0.033802 , 8 , 7 , 3 ], +[ 0.034029 , 9 , 8 , 3 ], +[ 0.034280 , 10 , 9 , 3 ], +[ 0.034288 , 10 , 9 , 4 ], +[ 0.034525 , 11 , 10 , 3 ], +[ 0.034533 , 11 , 10 , 4 ], +[ 0.034797 , 12 , 11 , 3 ], +[ 0.034989 , 13 , 12 , 3 ], +[ 0.035271 , 14 , 13 , 3 ], +[ 0.036008 , 17 , 16 , 3 ], +[ 0.036442 , 16 , 15 , 3 ], +[ 0.036596 , 19 , 0 , 3 ], +[ 0.036605 , 19 , 0 , 4 ], +[ 0.036749 , 20 , 1 , 3 ], +[ 0.036775 , 20 , 1 , 4 ], +[ 0.036787 , 20 , 1 , 5 ], +[ 0.036949 , 21 , 3 , 3 ], +[ 0.036958 , 21 , 3 , 4 ], +[ 0.037193 , 22 , 4 , 3 ], +[ 0.037215 , 22 , 4 , 4 ], +[ 0.037227 , 22 , 4 , 5 ], +[ 0.037820 , 24 , 0 , 3 ], +[ 0.037829 , 24 , 0 , 4 ], +[ 0.037973 , 25 , 1 , 3 ], +[ 0.037999 , 25 , 1 , 4 ], +[ 0.038011 , 25 , 1 , 5 ], +[ 0.038174 , 26 , 3 , 3 ], +[ 0.038183 , 26 , 3 , 4 ], +[ 0.038663 , 28 , 3 , 3 ], +[ 0.038672 , 28 , 3 , 4 ], +[ 0.039153 , 30 , 3 , 3 ], +[ 0.039162 , 30 , 3 , 4 ], +[ 0.039779 , 32 , 0 , 3 ], +[ 0.039788 , 32 , 0 , 4 ], +[ 0.039932 , 33 , 1 , 3 ], +[ 0.039958 , 33 , 1 , 4 ], +[ 0.039970 , 33 , 1 , 5 ], +[ 0.040133 , 34 , 3 , 3 ], +[ 0.040142 , 34 , 3 , 4 ], +[ 0.040656 , 36 , 2 , 3 ], +[ 0.040663 , 36 , 2 , 4 ], +[ 0.041111 , 38 , 4 , 3 ], +[ 0.041133 , 38 , 4 , 4 ], +[ 0.041145 , 38 , 4 , 5 ], +[ 0.041602 , 40 , 3 , 3 ], +[ 0.041611 , 40 , 3 , 4 ], +[ 0.043401 , 2 , 1 , 6 ], +[ 0.043434 , 1 , 0 , 5 ], +[ 0.043447 , 1 , 0 , 6 ], +[ 0.043815 , 3 , 2 , 5 ], +[ 0.043873 , 4 , 3 , 5 ], +[ 0.043886 , 4 , 3 , 6 ], +[ 0.044082 , 6 , 5 , 4 ], +[ 0.044570 , 7 , 6 , 4 ], +[ 0.044831 , 8 , 7 , 4 ], +[ 0.045090 , 5 , 4 , 6 ], +[ 0.045449 , 10 , 9 , 5 ], +[ 0.045694 , 11 , 10 , 5 ], +[ 0.045839 , 12 , 11 , 4 ], +[ 0.045998 , 9 , 8 , 4 ], +[ 0.046032 , 13 , 12 , 4 ], +[ 0.046300 , 14 , 13 , 4 ], +[ 0.046705 , 16 , 15 , 4 ], +[ 0.047031 , 17 , 16 , 4 ], +[ 0.047809 , 20 , 1 , 6 ], +[ 0.047842 , 19 , 0 , 5 ], +[ 0.047854 , 19 , 0 , 6 ], +[ 0.048035 , 21 , 3 , 5 ], +[ 0.048048 , 21 , 3 , 6 ], +[ 0.049033 , 25 , 1 , 6 ], +[ 0.049066 , 24 , 0 , 5 ], +[ 0.049079 , 24 , 0 , 6 ], +[ 0.049253 , 22 , 4 , 6 ], +[ 0.049260 , 26 , 3 , 5 ], +[ 0.049273 , 26 , 3 , 6 ], +[ 0.049749 , 28 , 3 , 5 ], +[ 0.049763 , 28 , 3 , 6 ], +[ 0.050239 , 30 , 3 , 5 ], +[ 0.050252 , 30 , 3 , 6 ], +[ 0.050992 , 33 , 1 , 6 ], +[ 0.051025 , 32 , 0 , 5 ], +[ 0.051038 , 32 , 0 , 6 ], +[ 0.051219 , 34 , 3 , 5 ], +[ 0.051232 , 34 , 3 , 6 ], +[ 0.051895 , 36 , 2 , 5 ], +[ 0.052688 , 40 , 3 , 5 ], +[ 0.052701 , 40 , 3 , 6 ], +[ 0.053171 , 38 , 4 , 6 ], +[ 0.054470 , 2 , 1 , 7 ], +[ 0.054487 , 2 , 1 , 8 ], +[ 0.054500 , 2 , 1 , 9 ], +[ 0.054521 , 1 , 0 , 7 ], +[ 0.054932 , 4 , 3 , 7 ], +[ 0.055104 , 6 , 5 , 5 ], +[ 0.055245 , 3 , 2 , 6 ], +[ 0.055432 , 8 , 7 , 5 ], +[ 0.055610 , 7 , 6 , 5 ], +[ 0.056678 , 10 , 9 , 6 ], +[ 0.056889 , 12 , 11 , 5 ], +[ 0.056923 , 11 , 10 , 6 ], +[ 0.057032 , 9 , 8 , 5 ], +[ 0.057064 , 13 , 12 , 5 ], +[ 0.057126 , 5 , 4 , 7 ], +[ 0.057323 , 14 , 13 , 5 ], +[ 0.058054 , 17 , 16 , 5 ], +[ 0.058877 , 20 , 1 , 7 ], +[ 0.058895 , 20 , 1 , 8 ], +[ 0.058907 , 20 , 1 , 9 ], +[ 0.058928 , 19 , 0 , 7 ], +[ 0.059094 , 21 , 3 , 7 ], +[ 0.060102 , 25 , 1 , 7 ], +[ 0.060119 , 25 , 1 , 8 ], +[ 0.060132 , 25 , 1 , 9 ], +[ 0.060153 , 24 , 0 , 7 ], +[ 0.060319 , 26 , 3 , 7 ], +[ 0.060372 , 16 , 15 , 5 ], +[ 0.060808 , 28 , 3 , 7 ], +[ 0.061288 , 22 , 4 , 7 ], +[ 0.061298 , 30 , 3 , 7 ], +[ 0.062061 , 33 , 1 , 7 ], +[ 0.062078 , 33 , 1 , 8 ], +[ 0.062091 , 33 , 1 , 9 ], +[ 0.062112 , 32 , 0 , 7 ], +[ 0.062278 , 34 , 3 , 7 ], +[ 0.063325 , 36 , 2 , 6 ], +[ 0.063747 , 40 , 3 , 7 ], +[ 0.065206 , 38 , 4 , 7 ], +[ 0.065542 , 2 , 1 , 10 ], +[ 0.065601 , 1 , 0 , 8 ], +[ 0.065614 , 1 , 0 , 9 ], +[ 0.065626 , 1 , 0 , 10 ], +[ 0.066011 , 4 , 3 , 8 ], +[ 0.066023 , 4 , 3 , 9 ], +[ 0.066036 , 4 , 3 , 10 ], +[ 0.066140 , 6 , 5 , 6 ], +[ 0.066484 , 8 , 7 , 6 ], +[ 0.066651 , 7 , 6 , 6 ], +[ 0.066656 , 3 , 2 , 7 ], +[ 0.067815 , 10 , 9 , 7 ], +[ 0.067928 , 12 , 11 , 6 ], +[ 0.068060 , 11 , 10 , 7 ], +[ 0.068094 , 13 , 12 , 6 ], +[ 0.068149 , 14 , 13 , 6 ], +[ 0.068160 , 5 , 4 , 8 ], +[ 0.068463 , 15 , 14 , 4 ], +[ 0.069950 , 20 , 1 , 10 ], +[ 0.070008 , 19 , 0 , 8 ], +[ 0.070022 , 19 , 0 , 9 ], +[ 0.070034 , 19 , 0 , 10 ], +[ 0.070173 , 21 , 3 , 8 ], +[ 0.070185 , 21 , 3 , 9 ], +[ 0.070199 , 21 , 3 , 10 ], +[ 0.071174 , 25 , 1 , 10 ], +[ 0.071233 , 24 , 0 , 8 ], +[ 0.071246 , 24 , 0 , 9 ], +[ 0.071258 , 24 , 0 , 10 ], +[ 0.071398 , 26 , 3 , 8 ], +[ 0.071410 , 26 , 3 , 9 ], +[ 0.071423 , 26 , 3 , 10 ], +[ 0.071888 , 28 , 3 , 8 ], +[ 0.071899 , 28 , 3 , 9 ], +[ 0.071913 , 28 , 3 , 10 ], +[ 0.072091 , 17 , 16 , 6 ], +[ 0.072322 , 22 , 4 , 8 ], +[ 0.072377 , 30 , 3 , 8 ], +[ 0.072389 , 30 , 3 , 9 ], +[ 0.072402 , 30 , 3 , 10 ], +[ 0.073133 , 33 , 1 , 10 ], +[ 0.073192 , 32 , 0 , 8 ], +[ 0.073205 , 32 , 0 , 9 ], +[ 0.073217 , 32 , 0 , 10 ], +[ 0.073357 , 34 , 3 , 8 ], +[ 0.073369 , 34 , 3 , 9 ], +[ 0.073382 , 34 , 3 , 10 ], +[ 0.074736 , 36 , 2 , 7 ], +[ 0.074826 , 40 , 3 , 8 ], +[ 0.074838 , 40 , 3 , 9 ], +[ 0.074851 , 40 , 3 , 10 ], +[ 0.076240 , 38 , 4 , 8 ], +[ 0.076607 , 2 , 1 , 11 ], +[ 0.076621 , 2 , 1 , 12 ], +[ 0.076633 , 2 , 1 , 13 ], +[ 0.076661 , 1 , 0 , 11 ], +[ 0.077065 , 4 , 3 , 11 ], +[ 0.077162 , 6 , 5 , 7 ], +[ 0.077509 , 8 , 7 , 7 ], +[ 0.077678 , 7 , 6 , 7 ], +[ 0.077745 , 3 , 2 , 8 ], +[ 0.078888 , 10 , 9 , 8 ], +[ 0.078959 , 12 , 11 , 7 ], +[ 0.079126 , 13 , 12 , 7 ], +[ 0.079133 , 11 , 10 , 8 ], +[ 0.079189 , 14 , 13 , 7 ], +[ 0.079212 , 5 , 4 , 9 ], +[ 0.081015 , 20 , 1 , 11 ], +[ 0.081028 , 20 , 1 , 12 ], +[ 0.081041 , 20 , 1 , 13 ], +[ 0.081068 , 19 , 0 , 11 ], +[ 0.081227 , 21 , 3 , 11 ], +[ 0.082239 , 25 , 1 , 11 ], +[ 0.082253 , 25 , 1 , 12 ], +[ 0.082265 , 25 , 1 , 13 ], +[ 0.082293 , 24 , 0 , 11 ], +[ 0.082452 , 26 , 3 , 11 ], +[ 0.082941 , 28 , 3 , 11 ], +[ 0.083374 , 22 , 4 , 9 ], +[ 0.083431 , 30 , 3 , 11 ], +[ 0.084198 , 33 , 1 , 11 ], +[ 0.084212 , 33 , 1 , 12 ], +[ 0.084224 , 33 , 1 , 13 ], +[ 0.084252 , 32 , 0 , 11 ], +[ 0.084411 , 34 , 3 , 11 ], +[ 0.084412 , 15 , 14 , 5 ], +[ 0.085546 , 15 , 14 , 6 ], +[ 0.085825 , 36 , 2 , 8 ], +[ 0.085880 , 40 , 3 , 11 ], +[ 0.086057 , 9 , 8 , 6 ], +[ 0.086065 , 9 , 8 , 7 ], +[ 0.086070 , 9 , 8 , 8 ], +[ 0.086120 , 17 , 16 , 7 ], +[ 0.086549 , 15 , 14 , 7 ], +[ 0.086558 , 15 , 14 , 8 ], +[ 0.087122 , 17 , 16 , 8 ], +[ 0.087127 , 17 , 16 , 9 ], +[ 0.087133 , 17 , 16 , 10 ], +[ 0.087138 , 17 , 16 , 11 ], +[ 0.087148 , 17 , 16 , 12 ], +[ 0.087292 , 38 , 4 , 9 ], +[ 0.087381 , 2 , 1 , 14 ], +[ 0.087410 , 2 , 1 , 15 ], +[ 0.087423 , 2 , 1 , 16 ], +[ 0.087727 , 1 , 0 , 12 ], +[ 0.087740 , 1 , 0 , 13 ], +[ 0.087753 , 1 , 0 , 14 ], +[ 0.088122 , 17 , 16 , 13 ], +[ 0.088152 , 4 , 3 , 12 ], +[ 0.088164 , 4 , 3 , 13 ], +[ 0.088176 , 4 , 3 , 14 ], +[ 0.088200 , 6 , 5 , 8 ], +[ 0.088521 , 8 , 7 , 8 ], +[ 0.088709 , 7 , 6 , 8 ], +[ 0.088855 , 3 , 2 , 9 ], +[ 0.088868 , 3 , 2 , 10 ], +[ 0.088880 , 3 , 2 , 11 ], +[ 0.088893 , 3 , 2 , 12 ], +[ 0.089129 , 17 , 16 , 14 ], +[ 0.089137 , 17 , 16 , 15 ], +[ 0.089142 , 17 , 16 , 16 ], +[ 0.089147 , 17 , 16 , 17 ], +[ 0.089152 , 17 , 16 , 18 ], +[ 0.089550 , 15 , 14 , 9 ], +[ 0.089559 , 15 , 14 , 10 ], +[ 0.089954 , 10 , 9 , 9 ], +[ 0.089997 , 12 , 11 , 8 ], +[ 0.090130 , 17 , 16 , 19 ], +[ 0.090135 , 17 , 16 , 20 ], +[ 0.090141 , 17 , 16 , 21 ], +[ 0.090169 , 13 , 12 , 8 ], +[ 0.090199 , 11 , 10 , 9 ], +[ 0.090232 , 14 , 13 , 8 ], +[ 0.091788 , 20 , 1 , 14 ], +[ 0.091818 , 20 , 1 , 15 ], +[ 0.091830 , 20 , 1 , 16 ], +[ 0.092134 , 19 , 0 , 12 ], +[ 0.092147 , 17 , 16 , 22 ], +[ 0.092148 , 19 , 0 , 13 ], +[ 0.092152 , 17 , 16 , 23 ], +[ 0.092160 , 19 , 0 , 14 ], +[ 0.092186 , 17 , 16 , 24 ], +[ 0.092192 , 17 , 16 , 25 ], +[ 0.092198 , 17 , 16 , 26 ], +[ 0.092272 , 5 , 4 , 10 ], +[ 0.092314 , 21 , 3 , 12 ], +[ 0.092326 , 21 , 3 , 13 ], +[ 0.092338 , 21 , 3 , 14 ], +[ 0.093013 , 25 , 1 , 14 ], +[ 0.093042 , 25 , 1 , 15 ], +[ 0.093055 , 25 , 1 , 16 ], +[ 0.093154 , 17 , 16 , 27 ], +[ 0.093160 , 17 , 16 , 28 ], +[ 0.093359 , 24 , 0 , 12 ], +[ 0.093372 , 24 , 0 , 13 ], +[ 0.093385 , 24 , 0 , 14 ], +[ 0.093539 , 26 , 3 , 12 ], +[ 0.093551 , 26 , 3 , 13 ], +[ 0.093563 , 26 , 3 , 14 ], +[ 0.094029 , 28 , 3 , 12 ], +[ 0.094040 , 28 , 3 , 13 ], +[ 0.094052 , 28 , 3 , 14 ], +[ 0.094518 , 30 , 3 , 12 ], +[ 0.094530 , 30 , 3 , 13 ], +[ 0.094542 , 30 , 3 , 14 ], +[ 0.094972 , 33 , 1 , 14 ], +[ 0.095001 , 33 , 1 , 15 ], +[ 0.095014 , 33 , 1 , 16 ], +[ 0.095318 , 32 , 0 , 12 ], +[ 0.095331 , 32 , 0 , 13 ], +[ 0.095344 , 32 , 0 , 14 ], +[ 0.095498 , 34 , 3 , 12 ], +[ 0.095510 , 34 , 3 , 13 ], +[ 0.095522 , 34 , 3 , 14 ], +[ 0.096434 , 22 , 4 , 10 ], +[ 0.096935 , 36 , 2 , 9 ], +[ 0.096948 , 36 , 2 , 10 ], +[ 0.096960 , 36 , 2 , 11 ], +[ 0.096967 , 40 , 3 , 12 ], +[ 0.096973 , 36 , 2 , 12 ], +[ 0.096979 , 40 , 3 , 13 ], +[ 0.096991 , 40 , 3 , 14 ], +[ 0.098100 , 9 , 8 , 9 ], +[ 0.098440 , 2 , 1 , 17 ], +[ 0.098777 , 1 , 0 , 15 ], +[ 0.098783 , 1 , 0 , 16 ], +[ 0.099207 , 4 , 3 , 15 ], +[ 0.099216 , 4 , 3 , 16 ], +[ 0.099234 , 6 , 5 , 9 ], +[ 0.099540 , 8 , 7 , 9 ], +[ 0.099733 , 7 , 6 , 9 ], +[ 0.099913 , 3 , 2 , 13 ], +[ 0.099918 , 3 , 2 , 14 ], +[ 0.100352 , 38 , 4 , 10 ], +[ 0.101031 , 10 , 9 , 10 ], +[ 0.101033 , 12 , 11 , 9 ], +[ 0.101218 , 13 , 12 , 9 ], +[ 0.101264 , 14 , 13 , 9 ], +[ 0.101276 , 11 , 10 , 10 ], +[ 0.102848 , 20 , 1 , 17 ], +[ 0.103184 , 19 , 0 , 15 ], +[ 0.103190 , 19 , 0 , 16 ], +[ 0.103369 , 21 , 3 , 15 ], +[ 0.103379 , 21 , 3 , 16 ], +[ 0.104072 , 25 , 1 , 17 ], +[ 0.104409 , 24 , 0 , 15 ], +[ 0.104415 , 24 , 0 , 16 ], +[ 0.104594 , 26 , 3 , 15 ], +[ 0.104603 , 26 , 3 , 16 ], +[ 0.105084 , 28 , 3 , 15 ], +[ 0.105093 , 28 , 3 , 16 ], +[ 0.105573 , 30 , 3 , 15 ], +[ 0.105582 , 30 , 3 , 16 ], +[ 0.106031 , 33 , 1 , 17 ], +[ 0.106368 , 32 , 0 , 15 ], +[ 0.106374 , 32 , 0 , 16 ], +[ 0.106553 , 34 , 3 , 15 ], +[ 0.106562 , 34 , 3 , 16 ], +[ 0.107993 , 36 , 2 , 13 ], +[ 0.107998 , 36 , 2 , 14 ], +[ 0.108022 , 40 , 3 , 15 ], +[ 0.108031 , 40 , 3 , 16 ], +[ 0.109544 , 2 , 1 , 18 ], +[ 0.109578 , 2 , 1 , 19 ], +[ 0.109590 , 2 , 1 , 20 ], +[ 0.109842 , 1 , 0 , 17 ], +[ 0.109854 , 1 , 0 , 18 ], +[ 0.109867 , 1 , 0 , 19 ], +[ 0.109879 , 1 , 0 , 20 ], +[ 0.109891 , 1 , 0 , 21 ], +[ 0.109904 , 1 , 0 , 22 ], +[ 0.110264 , 6 , 5 , 10 ], +[ 0.110291 , 4 , 3 , 17 ], +[ 0.110304 , 4 , 3 , 18 ], +[ 0.110316 , 4 , 3 , 19 ], +[ 0.110329 , 4 , 3 , 20 ], +[ 0.110341 , 4 , 3 , 21 ], +[ 0.110353 , 4 , 3 , 22 ], +[ 0.110567 , 8 , 7 , 10 ], +[ 0.110762 , 7 , 6 , 10 ], +[ 0.110992 , 3 , 2 , 15 ], +[ 0.111005 , 3 , 2 , 16 ], +[ 0.111017 , 3 , 2 , 17 ], +[ 0.111029 , 3 , 2 , 18 ], +[ 0.111042 , 3 , 2 , 19 ], +[ 0.111054 , 3 , 2 , 20 ], +[ 0.112076 , 12 , 11 , 10 ], +[ 0.112154 , 10 , 9 , 11 ], +[ 0.112274 , 13 , 12 , 10 ], +[ 0.112291 , 14 , 13 , 10 ], +[ 0.112399 , 11 , 10 , 11 ], +[ 0.113951 , 20 , 1 , 18 ], +[ 0.113986 , 20 , 1 , 19 ], +[ 0.113997 , 20 , 1 , 20 ], +[ 0.114249 , 19 , 0 , 17 ], +[ 0.114262 , 19 , 0 , 18 ], +[ 0.114274 , 19 , 0 , 19 ], +[ 0.114286 , 19 , 0 , 20 ], +[ 0.114299 , 19 , 0 , 21 ], +[ 0.114311 , 19 , 0 , 22 ], +[ 0.114453 , 21 , 3 , 17 ], +[ 0.114466 , 21 , 3 , 18 ], +[ 0.114478 , 21 , 3 , 19 ], +[ 0.114491 , 21 , 3 , 20 ], +[ 0.114503 , 21 , 3 , 21 ], +[ 0.114515 , 21 , 3 , 22 ], +[ 0.115148 , 9 , 8 , 10 ], +[ 0.115176 , 25 , 1 , 18 ], +[ 0.115210 , 25 , 1 , 19 ], +[ 0.115222 , 25 , 1 , 20 ], +[ 0.115474 , 24 , 0 , 17 ], +[ 0.115486 , 24 , 0 , 18 ], +[ 0.115499 , 24 , 0 , 19 ], +[ 0.115511 , 24 , 0 , 20 ], +[ 0.115523 , 24 , 0 , 21 ], +[ 0.115536 , 24 , 0 , 22 ], +[ 0.115678 , 26 , 3 , 17 ], +[ 0.115691 , 26 , 3 , 18 ], +[ 0.115703 , 26 , 3 , 19 ], +[ 0.115716 , 26 , 3 , 20 ], +[ 0.115728 , 26 , 3 , 21 ], +[ 0.115740 , 26 , 3 , 22 ], +[ 0.116167 , 28 , 3 , 17 ], +[ 0.116180 , 28 , 3 , 18 ], +[ 0.116192 , 28 , 3 , 19 ], +[ 0.116206 , 28 , 3 , 20 ], +[ 0.116217 , 28 , 3 , 21 ], +[ 0.116229 , 28 , 3 , 22 ], +[ 0.116657 , 30 , 3 , 17 ], +[ 0.116670 , 30 , 3 , 18 ], +[ 0.116682 , 30 , 3 , 19 ], +[ 0.116695 , 30 , 3 , 20 ], +[ 0.116707 , 30 , 3 , 21 ], +[ 0.116719 , 30 , 3 , 22 ], +[ 0.117135 , 33 , 1 , 18 ], +[ 0.117169 , 33 , 1 , 19 ], +[ 0.117181 , 33 , 1 , 20 ], +[ 0.117433 , 32 , 0 , 17 ], +[ 0.117445 , 32 , 0 , 18 ], +[ 0.117458 , 32 , 0 , 19 ], +[ 0.117470 , 32 , 0 , 20 ], +[ 0.117482 , 32 , 0 , 21 ], +[ 0.117495 , 32 , 0 , 22 ], +[ 0.117637 , 34 , 3 , 17 ], +[ 0.117650 , 34 , 3 , 18 ], +[ 0.117662 , 34 , 3 , 19 ], +[ 0.117675 , 34 , 3 , 20 ], +[ 0.117687 , 34 , 3 , 21 ], +[ 0.117699 , 34 , 3 , 22 ], +[ 0.119072 , 36 , 2 , 15 ], +[ 0.119085 , 36 , 2 , 16 ], +[ 0.119097 , 36 , 2 , 17 ], +[ 0.119106 , 40 , 3 , 17 ], +[ 0.119109 , 36 , 2 , 18 ], +[ 0.119119 , 40 , 3 , 18 ], +[ 0.119122 , 36 , 2 , 19 ], +[ 0.119131 , 40 , 3 , 19 ], +[ 0.119134 , 36 , 2 , 20 ], +[ 0.119144 , 40 , 3 , 20 ], +[ 0.119156 , 40 , 3 , 21 ], +[ 0.119168 , 40 , 3 , 22 ], +[ 0.120605 , 2 , 1 , 21 ], +[ 0.120900 , 1 , 0 , 23 ], +[ 0.120910 , 1 , 0 , 24 ], +[ 0.120914 , 1 , 0 , 25 ], +[ 0.121289 , 6 , 5 , 11 ], +[ 0.121346 , 4 , 3 , 23 ], +[ 0.121352 , 4 , 3 , 24 ], +[ 0.121357 , 4 , 3 , 25 ], +[ 0.121601 , 8 , 7 , 11 ], +[ 0.121787 , 7 , 6 , 11 ], +[ 0.122057 , 3 , 2 , 21 ], +[ 0.122062 , 3 , 2 , 22 ], +[ 0.122066 , 3 , 2 , 23 ], +[ 0.123109 , 12 , 11 , 11 ], +[ 0.123312 , 14 , 13 , 11 ], +[ 0.123322 , 13 , 12 , 11 ], +[ 0.125013 , 20 , 1 , 21 ], +[ 0.125308 , 19 , 0 , 23 ], +[ 0.125318 , 19 , 0 , 24 ], +[ 0.125322 , 19 , 0 , 25 ], +[ 0.125508 , 21 , 3 , 23 ], +[ 0.125514 , 21 , 3 , 24 ], +[ 0.125519 , 21 , 3 , 25 ], +[ 0.125629 , 15 , 14 , 11 ], +[ 0.126237 , 25 , 1 , 21 ], +[ 0.126532 , 24 , 0 , 23 ], +[ 0.126542 , 24 , 0 , 24 ], +[ 0.126546 , 24 , 0 , 25 ], +[ 0.126724 , 9 , 8 , 11 ], +[ 0.126733 , 26 , 3 , 23 ], +[ 0.126739 , 26 , 3 , 24 ], +[ 0.126744 , 26 , 3 , 25 ], +[ 0.127222 , 28 , 3 , 23 ], +[ 0.127229 , 28 , 3 , 24 ], +[ 0.127234 , 28 , 3 , 25 ], +[ 0.127712 , 30 , 3 , 23 ], +[ 0.127718 , 30 , 3 , 24 ], +[ 0.127723 , 30 , 3 , 25 ], +[ 0.128196 , 33 , 1 , 21 ], +[ 0.128491 , 32 , 0 , 23 ], +[ 0.128501 , 32 , 0 , 24 ], +[ 0.128505 , 32 , 0 , 25 ], +[ 0.128692 , 34 , 3 , 23 ], +[ 0.128698 , 34 , 3 , 24 ], +[ 0.128703 , 34 , 3 , 25 ], +[ 0.130137 , 36 , 2 , 21 ], +[ 0.130142 , 36 , 2 , 22 ], +[ 0.130146 , 36 , 2 , 23 ], +[ 0.130161 , 40 , 3 , 23 ], +[ 0.130167 , 40 , 3 , 24 ], +[ 0.130172 , 40 , 3 , 25 ], +[ 0.131565 , 1 , 0 , 26 ], +[ 0.131578 , 1 , 0 , 27 ], +[ 0.131590 , 1 , 0 , 28 ], +[ 0.131603 , 1 , 0 , 29 ], +[ 0.131615 , 1 , 0 , 30 ], +[ 0.131627 , 1 , 0 , 31 ], +[ 0.131640 , 1 , 0 , 32 ], +[ 0.131652 , 1 , 0 , 33 ], +[ 0.131664 , 1 , 0 , 34 ], +[ 0.131689 , 2 , 1 , 22 ], +[ 0.131702 , 2 , 1 , 23 ], +[ 0.131715 , 2 , 1 , 24 ], +[ 0.132022 , 4 , 3 , 26 ], +[ 0.132035 , 4 , 3 , 27 ], +[ 0.132047 , 4 , 3 , 28 ], +[ 0.132060 , 4 , 3 , 29 ], +[ 0.132072 , 4 , 3 , 30 ], +[ 0.132084 , 4 , 3 , 31 ], +[ 0.132097 , 4 , 3 , 32 ], +[ 0.132109 , 4 , 3 , 33 ], +[ 0.132387 , 6 , 5 , 12 ], +[ 0.132399 , 6 , 5 , 13 ], +[ 0.132412 , 6 , 5 , 14 ], +[ 0.132701 , 8 , 7 , 12 ], +[ 0.132714 , 8 , 7 , 13 ], +[ 0.132723 , 3 , 2 , 24 ], +[ 0.132727 , 8 , 7 , 14 ], +[ 0.132735 , 3 , 2 , 25 ], +[ 0.132739 , 8 , 7 , 15 ], +[ 0.132747 , 3 , 2 , 26 ], +[ 0.132751 , 8 , 7 , 16 ], +[ 0.132760 , 3 , 2 , 27 ], +[ 0.132764 , 8 , 7 , 17 ], +[ 0.132772 , 3 , 2 , 28 ], +[ 0.132784 , 3 , 2 , 29 ], +[ 0.132796 , 3 , 2 , 30 ], +[ 0.132809 , 3 , 2 , 31 ], +[ 0.132821 , 3 , 2 , 32 ], +[ 0.132907 , 7 , 6 , 12 ], +[ 0.132920 , 7 , 6 , 13 ], +[ 0.132932 , 7 , 6 , 14 ], +[ 0.132944 , 7 , 6 , 15 ], +[ 0.132957 , 7 , 6 , 16 ], +[ 0.132969 , 7 , 6 , 17 ], +[ 0.134138 , 12 , 11 , 12 ], +[ 0.134162 , 14 , 13 , 12 ], +[ 0.134377 , 13 , 12 , 12 ], +[ 0.135972 , 19 , 0 , 26 ], +[ 0.135986 , 19 , 0 , 27 ], +[ 0.135997 , 19 , 0 , 28 ], +[ 0.136011 , 19 , 0 , 29 ], +[ 0.136023 , 19 , 0 , 30 ], +[ 0.136034 , 19 , 0 , 31 ], +[ 0.136048 , 19 , 0 , 32 ], +[ 0.136059 , 19 , 0 , 33 ], +[ 0.136071 , 19 , 0 , 34 ], +[ 0.136096 , 20 , 1 , 22 ], +[ 0.136110 , 20 , 1 , 23 ], +[ 0.136122 , 20 , 1 , 24 ], +[ 0.136184 , 21 , 3 , 26 ], +[ 0.136197 , 21 , 3 , 27 ], +[ 0.136209 , 21 , 3 , 28 ], +[ 0.136222 , 21 , 3 , 29 ], +[ 0.136235 , 21 , 3 , 30 ], +[ 0.136246 , 21 , 3 , 31 ], +[ 0.136259 , 21 , 3 , 32 ], +[ 0.136271 , 21 , 3 , 33 ], +[ 0.137197 , 24 , 0 , 26 ], +[ 0.137210 , 24 , 0 , 27 ], +[ 0.137222 , 24 , 0 , 28 ], +[ 0.137235 , 24 , 0 , 29 ], +[ 0.137247 , 24 , 0 , 30 ], +[ 0.137259 , 24 , 0 , 31 ], +[ 0.137272 , 24 , 0 , 32 ], +[ 0.137284 , 24 , 0 , 33 ], +[ 0.137296 , 24 , 0 , 34 ], +[ 0.137321 , 25 , 1 , 22 ], +[ 0.137334 , 25 , 1 , 23 ], +[ 0.137347 , 25 , 1 , 24 ], +[ 0.137409 , 26 , 3 , 26 ], +[ 0.137422 , 26 , 3 , 27 ], +[ 0.137434 , 26 , 3 , 28 ], +[ 0.137447 , 26 , 3 , 29 ], +[ 0.137459 , 26 , 3 , 30 ], +[ 0.137471 , 26 , 3 , 31 ], +[ 0.137484 , 26 , 3 , 32 ], +[ 0.137496 , 26 , 3 , 33 ], +[ 0.137898 , 28 , 3 , 26 ], +[ 0.137911 , 28 , 3 , 27 ], +[ 0.137924 , 28 , 3 , 28 ], +[ 0.137936 , 28 , 3 , 29 ], +[ 0.137949 , 28 , 3 , 30 ], +[ 0.137960 , 28 , 3 , 31 ], +[ 0.137973 , 28 , 3 , 32 ], +[ 0.137986 , 28 , 3 , 33 ], +[ 0.138388 , 30 , 3 , 26 ], +[ 0.138401 , 30 , 3 , 27 ], +[ 0.138413 , 30 , 3 , 28 ], +[ 0.138426 , 30 , 3 , 29 ], +[ 0.138438 , 30 , 3 , 30 ], +[ 0.138450 , 30 , 3 , 31 ], +[ 0.138463 , 30 , 3 , 32 ], +[ 0.138475 , 30 , 3 , 33 ], +[ 0.138784 , 9 , 8 , 12 ], +[ 0.139156 , 32 , 0 , 26 ], +[ 0.139169 , 32 , 0 , 27 ], +[ 0.139181 , 32 , 0 , 28 ], +[ 0.139194 , 32 , 0 , 29 ], +[ 0.139206 , 32 , 0 , 30 ], +[ 0.139218 , 32 , 0 , 31 ], +[ 0.139231 , 32 , 0 , 32 ], +[ 0.139243 , 32 , 0 , 33 ], +[ 0.139255 , 32 , 0 , 34 ], +[ 0.139280 , 33 , 1 , 22 ], +[ 0.139293 , 33 , 1 , 23 ], +[ 0.139306 , 33 , 1 , 24 ], +[ 0.139368 , 34 , 3 , 26 ], +]; + +sfr_names=["http_get", + "http_post", + "https", + "http_browsing", + "exchange", + "mail_pop", + "mail_pop_1", + "mail_pop_2", + "oracle_0", + "rtp_160k", + "rtp_250k", + "smtp_0", + "smtp_1", + "smtp_2", + "video_call", + "sip_video_call_full", + "citrix_0", + "dns_0"]; + +chart("#chart4",sfr_data,sfr_names,"time-sec","flow-id"); +</script> ++++++++++++++++++++++++++++++++++ + +== TRex command line main options + +[source,bash] +---- +$.sudo /t-rex-64 -f [traffic_yaml] -m [muti] -d [duration] -l [Hz=1000] -c [cores] +---- + +*-f=TRAFFIC_CFG* :: + Traffic YAML configuration file + +*-m*:: + CPS multiplier + +*-d=DURATION* :: + Duration of the test in sec + +*-l=Hz* :: + Latency pakets rate (Hz) + +*-c*:: + How many cores to use + + +== TRex Vision + +* All in One, most of the functionality of existing tools +* Standard tests (e.g. traffic mix, automation) across features/platforms +* Cisco wide developers community +* Open source, Standardize tests + +== Trex GUI + +image:images/TrexViewer.png[width=500] + +* Monitor GUI works on Windows 7 +* Nonitor real-time properties of TRex ( e.g min/max/average latency,jitter ) + +== Python API and automation tools + +++++++++++++++++++ +<img src="images/trex_control_plane_modules.png" alt="title=" align="center" > + +<div id="pclient" > + +++++++++++++++++++ + +[source,python] +----- +import trex_client + + trex = trex_client.CTRexClient('csi-kiwi-02') + ret = trex.start_trex( + m = 1.1, + d = 20, + f = 'avl/sfr_delay_10_1g.yaml', + l = 1000) + + while trex.is_running() : + trex.get_running_info() + res=trex.get_result_obj() + if res.is_done_warmup (): + res.get_value_list("trex-global.data.m_tx_bps"); + time.sleep(1); + + ret = trex.stop_trex() +----- + +++++++++++++++++++ +</div> + +<script> + $("#pclient").css({'font-size':'12px'}) + </script> +++++++++++++++++++ + + +== Trex On your laptop + +++++++++++++++++++ +<img src="images/T-Rex_vm.png" alt="title=" align="center" > +++++++++++++++++++ + +* You can experience TRex on your laptop + +== Roadmap + +* TCP stack +* Export SDK application plugins API - more application support +* Automation (e.g ATS, Cisco HLTAPI etc) +* Open source +* Tunnel agnostic (e.g. VXLAN,QinQ,MPLS,NSH) + +== Benefit + +** Significant budget saving +** Product quality + +== Resource + +.Resource +[options="header",cols="1^",width="40%"] +|================= +| Link +| link:trex_manual.html[Manual-html] +| link:trex_book.pdf[Manual-pdf] +| link:release_notes.html[Release Notes] +| link:trex_manual.html#_download_and_installation[How to install] +|================= + + +++++++++++++++++++ +</section> +<section class="slide"> +<h2></h2> +<h7>Backup</h7> +<h8></h8> +</section> +++++++++++++++++++ + + + + diff --git a/trex_vm_manual.asciidoc b/trex_vm_manual.asciidoc new file mode 100755 index 00000000..3a1e62d6 --- /dev/null +++ b/trex_vm_manual.asciidoc @@ -0,0 +1,324 @@ +T-Rex Virtual Machine setup and basic usage +=========================================== +:author: Dan Klein +:email: <danklei@cisco.com> +:revnumber: 1.0 +:quotes.++: +:numbered: + + +== Introduction + +=== T-Rex traffic generator + +T-Rex traffic generator is a tool design the benchmark platforms with realistic traffic. +This is a work-in-progress product, which is under constant developement, new features are added and support for more router's fuctionality is achieved. + +One of the innovative tools through which T-Rex can be learned and tested during development is a virtual machine instance, fully simulating T-Rex without the need of any additional hardware. + + +==== T-Rex Virtual Machine + +The T-Rex Virtual Machine is based on Oracle's Virtual Box freeware. + +It is designed to enable T-Rex newbies to explore this tool without any special resources, and no single network cable attached. Moreover, is requires no internet or outer network connectivity. + + +== Setup and Usage + +=== Setup + +In order to use T-Rex VM, there are several super-easy steps to follow: + + . Download and install Oracle VM Virtual Box Manage https://www.virtualbox.org/wiki/Downloads[(VB download link)]. + + During the installation you'll be asked to allow the installation of system devices component interactions. Allow it. + . Download the latest T-Rex VM by http://csi-wiki-01:8181/trex/T_Rex_162_VM_Fedora_21.ova[clicking on this link]. + . Open Oracle VM Virtual Box application installed at step 1. + . Under 'File' tab, select 'Import Appliance' (`ctrl+I` shorthand can be used as well). The following screen will apear: + +ifdef::backend-docbook[] +image::images/vm_import.png[title="VM import screen",align="center",width=400, link="images/vm_import.png"] +endif::backend-docbook[] + +ifdef::backend-xhtml11[] +image::images/vm_import.png[title="VM import screen",align="center",width=900, link="images/vm_import.png"] +endif::backend-xhtml11[] + + + + . Browse and select the .ova file you have downloaded at step 2, and click 'continue'. + . Click 'Next, and then make sure that the 'Reinitialize the MAC address of all network cards' checkbox is **not selected**. + . Click 'import' and wait for the import process to finish. + . **That's it! you're all good and set to go!** + + +=== Launching and logging into the machine + +Let's get started with running T-Rex! + +First, launch the virtual machine by selecting it in the VM's menu and hitting 'Start' button. + +ifdef::backend-docbook[] +image::images/vm_selection_screen.png[title="T-Rex VM launching screen",align="center",width=400, link="images/vm_selection_screen.png"] +endif::backend-docbook[] + +ifdef::backend-xhtml11[] +image::images/vm_selection_screen.png[title="T-Rex VM launching screen",align="center",width=900, link="images/vm_selection_screen.png"] +endif::backend-xhtml11[] + + + + +[IMPORTANT] +==== +You may encounter with "VT-x is disabled" error, similar to the image below. + +In that case, please refer to https://www.virtualbox.org/ticket/4130[this link] and follow the provided steps to overcome this issue. +==== + +ifdef::backend-docbook[] +image::images/trex_vm_bios_err.png[title="VT-x disabled possible error message",align="center",width=400, link="images/trex_vm_bios_err.png"] +endif::backend-docbook[] + +ifdef::backend-xhtml11[] +image::images/trex_vm_bios_err.png[title="VT-x disabled possible error message",align="center",width=900, link="images/trex_vm_bios_err.png"] +endif::backend-xhtml11[] + + +<<<<< + +Next, once the machine is finished booting, login to the machine using the following credentials: + + - Username: `trex` + + - Password: `trex` + +ifdef::backend-docbook[] +image::images/trex_vm_login.png[title="T-Rex VM login",align="center",width=400, link="images/trex_vm_login.png"] +endif::backend-docbook[] + +ifdef::backend-xhtml11[] +image::images/trex_vm_login.png[title="T-Rex VM login",align="center",width=900, link="images/trex_vm_login.png"] +endif::backend-xhtml11[] + + +TIP: a remote connection to the machine from anywhere in the hosting machine can be achieved by the following command: + + `ssh -p 3022 trex@127.0.0.1` + +=== Running T-Rex traffic generator + +Finally, we're ready to do some T-Rex magic. This is super easy and can be achived be the following steps: + + 1. Change dir to latest version supplied using: `cd /home/trex/v1.62/`. + + 2. Run your desired T-Rex command. + +IMPORTANT: When launching a T-Rex command pay attention make sure that: + + 1. use `sudo` prefix at the beggining of the command. + + 2. Specify the `-f` argument first among all arguments. + + +For example, let's run T-Rex with DNS traffic generation and no latency. The Runnning command is: + +---- +[trex@localhost v1.62]$ sudo ./t-rex-64 -f cap2/dns.yaml -d 100 -m 1 --nc +Starting T-Rex 1.62 please wait ... +found configuration file at /etc/trex_cfg.yaml +zmq publisher at: tcp://*:4500 + +... +<1> +... + +-Per port stats table + ports | 0 | 1 + ----------------------------------------------------------------------------------------- + opackets | 17 | 17 + obytes | 1241 | 1513 + ipackets | 17 | 17 + ibytes | 1513 | 1241 + ierrors | 0 | 0 + oerrors | 0 | 0 + Tx Bw | 582.35 bps | 709.99 bps + +-Global stats enabled + Cpu Utilization : 0.8 % 0.0 Gb/core + Platform_factor : 1.0 + Total-Tx : 1.29 Kbps + Total-Rx : 1.29 Kbps + Total-PPS : 1.99 pps + Total-CPS : 1.00 cps + + Expected-PPS : 2.00 pps + Expected-CPS : 1.00 cps + Expected-BPS : 1.30 Kbps + + Active-flows : 0 Clients : 511 Socket-util : 0.0001 % + Open-flows : 17 Servers : 255 Socket : 17 Socket/Clients : 0.0 + drop-rate : 0.00 bps + current time : 18.7 sec + test duration : 81.3 sec +---- +<1> Output prompt continues (trimmed for reader's comfort). + +Now, lets review the generated packets as they observed footnoteref:[<1>,The given output represents the output begining.] by our promiscuous interface (interface #2): + +---- +[trex@localhost ~]$ sudo tcpdump -i enp0s8 +tcpdump: verbose output suppressed, use -v or -vv for full protocol decode +listening on enp0s8, link-type EN10MB (Ethernet), capture size 262144 bytes +09:38:53.953651 IP 16.0.0.2.1024 > 48.0.0.2.domain: 48 A? www.cisco.com. (31) +09:38:53.963969 IP 48.0.0.2.domain > 16.0.0.2.1024: 48* 1/0/0 A 100.100.100.100 (47) +09:38:54.960361 IP 16.0.0.3.1024 > 48.0.0.3.domain: 48 A? www.cisco.com. (31) +09:38:54.970358 IP 48.0.0.3.domain > 16.0.0.3.1024: 48* 1/0/0 A 100.100.100.100 (47) +09:38:55.967200 IP 16.0.0.4.1024 > 48.0.0.4.domain: 48 A? www.cisco.com. (31) +09:38:55.977222 IP 48.0.0.4.domain > 16.0.0.4.1024: 48* 1/0/0 A 100.100.100.100 (47) +09:38:56.975355 IP 16.0.0.5.1024 > 48.0.0.5.domain: 48 A? www.cisco.com. (31) +09:38:56.985379 IP 48.0.0.5.domain > 16.0.0.5.1024: 48* 1/0/0 A 100.100.100.100 (47) +09:38:57.981659 IP 16.0.0.6.1024 > 48.0.0.6.domain: 48 A? www.cisco.com. (31) +09:38:57.992358 IP 48.0.0.6.domain > 16.0.0.6.1024: 48* 1/0/0 A 100.100.100.100 (47) +09:38:58.990979 IP 16.0.0.7.1024 > 48.0.0.7.domain: 48 A? www.cisco.com. (31) +09:38:59.000952 IP 48.0.0.7.domain > 16.0.0.7.1024: 48* 1/0/0 A 100.100.100.100 (47) +09:39:00.009403 IP 16.0.0.8.1024 > 48.0.0.8.domain: 48 A? www.cisco.com. (31) +09:39:00.019456 IP 48.0.0.8.domain > 16.0.0.8.1024: 48* 1/0/0 A 100.100.100.100 (47) +09:39:01.015810 IP 16.0.0.9.1024 > 48.0.0.9.domain: 48 A? www.cisco.com. (31) +---- + + +Let's have a look at another example. + +We want to run T-Rex with simple http traffic generation. The running command will look like this: + +---- +[trex@localhost v1.62]$ sudo ./t-rex-64 -f cap2/http_simple.yaml -d 100 -l 1000 -m 1 --nc +Starting T-Rex 1.62 please wait ... +found configuration file at /etc/trex_cfg.yaml +zmq publisher at: tcp://*:4500 + +... +<1> +... + +-Per port stats table + ports | 0 | 1 + ----------------------------------------------------------------------------------------- + opackets | 40983 | 41946 + obytes | 2563951 | 6015664 + ipackets | 41946 | 40983 + ibytes | 6015664 | 2563951 + ierrors | 0 | 0 + oerrors | 0 | 0 + Tx Bw | 520.83 Kbps | 1.27 Mbps + +-Global stats enabled + Cpu Utilization : 3.1 % 0.1 Gb/core + Platform_factor : 1.0 + Total-Tx : 1.79 Mbps + Total-Rx : 1.79 Mbps + Total-PPS : 2.11 Kpps + Total-CPS : 2.84 cps + + Expected-PPS : 102.71 pps + Expected-CPS : 2.78 cps + Expected-BPS : 764.51 Kbps + + Active-flows : 0 Clients : 255 Socket-util : 0.0000 % + Open-flows : 107 Servers : 65535 Socket : 0 Socket/Clients : 0.0 + drop-rate : 0.00 bps + current time : 39.6 sec + test duration : 60.4 sec + +-Latency stats enabled + Cpu Utilization : 1.0 % + if| tx_ok , rx_ok , rx ,error, average , max , Jitter , max window + | , , check, , latency(usec),latency (usec) ,(usec) , + ---------------------------------------------------------------------------------------- + 0 | 39490, 39489, 0, 0, 1276 , 106714, 91 | 1737 1880 + 1 | 39490, 39490, 0, 0, 226 , 107619, 203 | 1694 1041 + +---- + +<1> Output prompt continues (trimmed for reader's comfort). + +Once again, lets review the generated packets as they observed footnoteref:[<1>] by our promiscuous interface (interface #2): + +---- +[trex@localhost ~]$ sudo tcpdump -a -i enp0s8 +tcpdump: verbose output suppressed, use -v or -vv for full protocol decode +listening on enp0s8, link-type EN10MB (Ethernet), capture size 262144 bytes +19:31:46.650426 IP 16.0.0.1.1024 > 48.0.0.1.http: Flags [S], seq 404375002, win 32768, options [mss 1460], length 0 +19:31:46.650439 IP 16.0.0.1.17 > 48.0.0.1.80: sctp (1) [Bad chunk length 0] +19:31:46.650442 IP 48.0.0.1.17 > 16.0.0.1.80: sctp (1) [Bad chunk length 0] +19:31:46.650445 IP 16.0.0.1.17 > 48.0.0.1.80: sctp (1) [Bad chunk length 0] +19:31:46.652458 IP 48.0.0.1.17 > 16.0.0.1.80: sctp (1) [Bad chunk length 0] +19:31:46.652462 IP 16.0.0.1.17 > 48.0.0.1.80: sctp (1) [Bad chunk length 0] +19:31:46.652465 IP 48.0.0.1.17 > 16.0.0.1.80: sctp (1) [Bad chunk length 0] +19:31:47.152768 IP 16.0.0.1.17 > 48.0.0.1.80: sctp (1) [Bad chunk length 0] +19:31:47.152788 IP 48.0.0.1.17 > 16.0.0.1.80: sctp (1) [Bad chunk length 0] +19:31:47.153796 IP 16.0.0.1.17 > 48.0.0.1.80: sctp (1) [Bad chunk length 0] +19:31:47.153801 IP 48.0.0.1.17 > 16.0.0.1.80: sctp (1) [Bad chunk length 0] +19:31:47.154803 IP 48.0.0.2.http > 16.0.0.2.1024: Flags [P.], seq 404419110:404420570, ack 404375252, win 32768, length 1460 +19:31:47.154823 IP 48.0.0.2.http > 16.0.0.2.1024: Flags [P.], seq 1460:2920, ack 1, win 32768, length 1460 +---- + +[NOTE] +See http://csi-wiki-01:8181/trex/doc/trex_book.pdf[T-Rex full manual] for a complete understading of the tool features and options. + + +=== T-Rex Live monitoring + +Once we have T-Rex up and running, we can enjoy the benefit of having live monitoring on its performance, using TRexViewer application footnote:[Supported only on Windows OS] + + + +This can be easily done by following these steps: + 0. Download the latest version of TrexViewer application and install it using http://csi-wiki-01:8080/display/bpsim/TrexViewer[this link]. + + 1. Start the application and fill in the following: + + - Trex ip: `127.0.0.1:4500` + + 2. Click the play button. + +ifdef::backend-docbook[] +image::images/trex_motinor_config.png[title="T-Rex viewer start screen",align="center",width=400,link="images/trex_motinor_config.png"] +endif::backend-docbook[] + +ifdef::backend-xhtml11[] +image::images/trex_motinor_config.png[title="T-Rex viewer start screen",align="center",width=900,link="images/trex_motinor_config.png"] +endif::backend-xhtml11[] + + + + 3. **That's it!** + + Now the live data from T-Rex will be displayed on the screen. + +ifdef::backend-docbook[] +image::images/trex_motinor_view.png[title="T-Rex viewer monitor screen",align="center",width=400,link="images/trex_motinor_view.png"] +endif::backend-docbook[] + +ifdef::backend-xhtml11[] +image::images/trex_motinor_view.png[title="T-Rex viewer monitor screen",align="center",width=900,link="images/trex_motinor_view.png"] +endif::backend-xhtml11[] + + +[NOTE] +Make sure T-Rex is running, otherwise data will not be available at TRexViewer. + +=== Architecture and network design + +Since no hardware is used, T-Rex simulates traffic using a virtual internal network, named 'trex_intnet'. + +The following figure describes the virtual "wiring" of the virtual machine to support T-Rex traffic simulation. + +ifdef::backend-docbook[] +image::images/T-Rex_vm.png[title="T-Rex virtual connectivity",align="center",width=400, link="images/T-Rex_vm.png"] +endif::backend-docbook[] + +ifdef::backend-xhtml11[] +image::images/T-Rex_vm.png[title="T-Rex virtual connectivity",align="center",width=900, link="images/T-Rex_vm.png"] +endif::backend-xhtml11[] + + +The VM runs T-Rex with single client and single server port. The traffic generated by each of those ports are switched over the 'trex_intnet' virtual network and received by the other side. + +T-Rex identifies only the packets which were dedicately sent by one of those traffic ports and receives them in the other port. Hence, packets generated by client port will be received by the server port and vice versa. + +Ontop, network adapter #4 used to [underline]#listen# to all traffic generated by both of T-Rex's ports, therefore it is very useful in providing live data of the generated flows over the network.
\ No newline at end of file Binary files differdiff --git a/waf.css b/waf.css new file mode 100755 index 00000000..e9fa3f50 --- /dev/null +++ b/waf.css @@ -0,0 +1,39 @@ +div.tableblock > table { + border: 1px solid gray; +} + +div#header-pic { + background-image: url("images/bg4.jpg"); + background-repeat: no-repeat; + background-color: #cccccc; +} + +a:visited { + color: #8A2908; +} + + +div#header h1 { + background: url('images/trex_logo_64_64.png') no-repeat left center; + padding-left: 80px; + line-height: 80px; + height: 80px; +} + +div.title, caption.title { + text-align: center; + margin-bottom: 0.2em; +} + +div.tableblock > table th { + background-color: #F4F4F4; +} + +h1, h2, h3, h4, h5, h6, span#author, div.title, caption.title, div.admonitionblock .icon, div#toctitle, div.sidebar-title, div.image-title { + color: #333; +} + +body, div.sectionbody, div#toctitle { + font-family: 'Lucida Grande', Verdana, Arial, sans-serif; +} + diff --git a/wscript b/wscript new file mode 100755 index 00000000..b4fe280c --- /dev/null +++ b/wscript @@ -0,0 +1,180 @@ +#! /usr/bin/env python +# encoding: utf-8 +# hhaim, 2014 (IL) base on WAF book + +""" +call 'waf --targets=waf.pdf' or use 'waf list' to see the targets available +""" + +VERSION='0.0.1' +APPNAME='wafdocs' + +import os, re, shutil + + +top = '.' +out = 'build' + +re_xi = re.compile('''^(include|image)::([^.]*.(asciidoc|\\{PIC\\}))\[''', re.M) +def ascii_doc_scan(self): + p = self.inputs[0].parent + node_lst = [self.inputs[0]] + seen = [] + depnodes = [] + while node_lst: + nd = node_lst.pop(0) + if nd in seen: continue + seen.append(nd) + + code = nd.read() + for m in re_xi.finditer(code): + name = m.group(2) + if m.group(3) == '{PIC}': + + ext = '.eps' + if self.generator.rule.rfind('A2X') > 0: + ext = '.png' + + k = p.find_resource(name.replace('{PIC}', ext)) + if k: + depnodes.append(k) + else: + k = p.find_resource(name) + if k: + depnodes.append(k) + node_lst.append(k) + return [depnodes, ()] + + + +import re +def scansize(self): + name = 'image::%s\\{PIC\\}\\[.*,(width|height)=(\\d+)' % self.inputs[0].name[:-4] + re_src = re.compile(name) + lst = self.inputs[0].parent.get_src().ant_glob('*.txt') + for x in lst: + m = re_src.search(x.read()) + if m: + val = str(int(1.6 * int(m.group(2)))) + if m.group(1) == 'width': + w = val + h = "800" + else: + w = "800" + h = val + + ext = self.inputs[0].name[-3:] + if ext == 'eps': + code = '-geometry %sx%s' % (w, h) + elif ext == 'dia': + if m.group(1) == 'width': + h = '' + else: + w = '' + code = '--size %sx%s' % (w, h) + else: + code = '-Gsize="%s,%s"' % (w, h) + break + else: + return ([], '') + + return ([], code) + +def options(opt): + opt.add_option('--exe', action='store_true', default=False, help='Execute the program after it is compiled') + +def configure(conf): + conf.find_program('asciidoc', path='/usr/local/bin/', var='ASCIIDOC') + pass; + +def convert_to_pdf(task): + input_file = task.outputs[0].abspath() + out_dir = task.outputs[0].parent.get_bld().abspath() + os.system('a2x --no-xmllint -v -f pdf -d article %s -D %s ' %(task.inputs[0].abspath(),out_dir ) ) + return (0) + +def convert_to_pdf_book(task): + input_file = task.outputs[0].abspath() + out_dir = task.outputs[0].parent.get_bld().abspath() + os.system('a2x --no-xmllint -v -f pdf -d book %s -D %s ' %(task.inputs[0].abspath(),out_dir ) ) + return (0) + + +def ensure_dir(f): + if not os.path.exists(f): + os.makedirs(f) + + +def my_copy(task): + input_file=task.outputs[0].abspath() + out_dir=task.outputs[0].parent.get_bld().abspath() + ensure_dir(out_dir) + shutil.copy2(input_file, out_dir+ os.sep+task.outputs[0].name) + return (0) + + +def do_visio(bld): + for x in bld.path.ant_glob('visio\\*.vsd'): + tg = bld(rule='${VIS} -i ${SRC} -o ${TGT} ', source=x, target=x.change_ext('.png')) + +def build(bld): + bld(rule=my_copy, target='symbols.lang') + + for x in bld.path.ant_glob('images\\**\**.png'): + bld(rule=my_copy, target=x) + bld.add_group() + + for x in bld.path.ant_glob('video\\**\**.mp4'): + bld(rule=my_copy, target=x) + bld.add_group() + + + for x in bld.path.ant_glob('images\\**\**.jpg'): + bld(rule=my_copy, target=x) + bld.add_group() + + bld(rule=my_copy, target='my_chart.js') + + bld.add_group() # separator, the documents may require any of the pictures from above + + bld(rule='${ASCIIDOC} -b deckjs -o ${TGT} ${SRC[0].abspath()}', + source='trex_config.asciidoc ', target='trex_config_guide.html', scan=ascii_doc_scan) + + + bld(rule='${ASCIIDOC} -b deckjs -o ${TGT} ${SRC[0].abspath()}', + source='trex_preso.asciidoc ', target='trex_preso.html', scan=ascii_doc_scan) + + bld(rule='${ASCIIDOC} -a stylesheet=${SRC[1].abspath()} -a icons=true -a max-width=55em -o ${TGT} ${SRC[0].abspath()}', + source='release_notes.asciidoc waf.css', target='release_notes.html', scan=ascii_doc_scan) + + + bld(rule='${ASCIIDOC} -a docinfo -a stylesheet=${SRC[1].abspath()} -a icons=true -a toc2 -a max-width=55em -d book -o ${TGT} ${SRC[0].abspath()}', + source='trex_book.asciidoc waf.css', target='trex_manual.html', scan=ascii_doc_scan) + + bld(rule=convert_to_pdf_book, + source='trex_book.asciidoc waf.css', target='trex_book.pdf', scan=ascii_doc_scan) + + + bld(rule=convert_to_pdf_book, + source='trex_vm_manual.asciidoc waf.css', target='trex_vm_manual.pdf', scan=ascii_doc_scan) + + bld(rule=convert_to_pdf_book, + source='trex_control_plane_peek.asciidoc waf.css', target='trex_control_plane_peek.pdf', scan=ascii_doc_scan) + + bld(rule=convert_to_pdf_book, + source='trex_control_plane_design_phase1.asciidoc waf.css', target='trex_control_plane_design_phase1.pdf', scan=ascii_doc_scan) + + bld(rule='${ASCIIDOC} -a stylesheet=${SRC[1].abspath()} -a icons=true -a toc2 -a max-width=55em -o ${TGT} ${SRC[0].abspath()}', + source='trex_vm_manual.asciidoc waf.css', target='trex_vm_manual.html', scan=ascii_doc_scan) + + bld(rule='${ASCIIDOC} -a stylesheet=${SRC[1].abspath()} -a icons=true -a toc2 -a max-width=55em -o ${TGT} ${SRC[0].abspath()}', + source='trex_control_plane_design_phase1.asciidoc waf.css', target='trex_control_plane_design_phase1.html', scan=ascii_doc_scan) + + bld(rule='${ASCIIDOC} -a stylesheet=${SRC[1].abspath()} -a icons=true -a toc2 -a max-width=55em -o ${TGT} ${SRC[0].abspath()}', + source='trex_control_plane_peek.asciidoc waf.css', target='trex_control_plane_peek.html', scan=ascii_doc_scan) + + + + + + |