diff options
| -rw-r--r-- | docs/about.rst | 4 | ||||
| -rw-r--r-- | docs/gettingstarted/developers/fib20/index.rst | 1 | ||||
| -rw-r--r-- | docs/gettingstarted/developers/fib20/scale.rst | 283 | ||||
| -rw-r--r-- | src/vnet/fib/fib_node.c | 2 | ||||
| -rw-r--r-- | src/vnet/fib/ip4_fib.c | 4 | ||||
| -rw-r--r-- | src/vnet/fib/ip6_fib.c | 31 | ||||
| -rw-r--r-- | src/vnet/fib/mpls_fib.c | 2 | ||||
| -rw-r--r-- | src/vnet/mfib/ip4_mfib.c | 2 | ||||
| -rw-r--r-- | src/vnet/mfib/ip6_mfib.c | 9 |
9 files changed, 320 insertions, 18 deletions
diff --git a/docs/about.rst b/docs/about.rst index 277a2fbf126..65a66a95610 100644 --- a/docs/about.rst +++ b/docs/about.rst @@ -4,6 +4,6 @@ About ===== -**VPP Version:** 19.08-rc0~300-g9080096f7 +**VPP Version:** 19.08-rc0~307-g883a867 -**Built on:** Tue May 28 13:45:28 GMT 2019 +**Built on:** Wed May 29 06:27:33 GMT 2019 diff --git a/docs/gettingstarted/developers/fib20/index.rst b/docs/gettingstarted/developers/fib20/index.rst index 535e15662d6..1b014524cb8 100644 --- a/docs/gettingstarted/developers/fib20/index.rst +++ b/docs/gettingstarted/developers/fib20/index.rst @@ -11,3 +11,4 @@ FIB 2.0 Hierarchical, Protocol, Independent mplsfib multicast fastconvergence + scale diff --git a/docs/gettingstarted/developers/fib20/scale.rst b/docs/gettingstarted/developers/fib20/scale.rst new file mode 100644 index 00000000000..c97297d4adc --- /dev/null +++ b/docs/gettingstarted/developers/fib20/scale.rst @@ -0,0 +1,283 @@ +.. _scale: + +Scale +----- + +The only limiting factor on FIB scale is the amount of memory +allocated to each heap the FIB uses, and there are 4: + +* The IP4 heap +* The IP6 heap +* The main heap +* The stats heap + + +IP4 Heap +-------- + +The IPv4 heap is used to allocate the memory needed for the +data-structures within which the IPv4 prefixes are stored. Each +table, created by the user, i.e. with; + +.. code-block:: console + + $ ip table add 1 + +or the default table, comprises 2 *ip4_fib_t* objects. +The 'non-forwarding' *ip4_fib_t* contains all the entries in the table +and, the 'forwarding' contains the entries that are matched against in +the data-plane. The difference between the two sets are the entries +that should not be matched in the data-plane. +Each *ip4_fib_t* comprises an mtrie (for fast lookup in the data-plane) +and a hash table per-prefix length (for lookup in the control plane). + +To see the amount of memory consumed by the IPv4 tables use: + +.. code-block:: console + + vpp# sh ip fib mem + ipv4-VRF:0 mtrie:333056 hash:3523 + ipv4-VRF:1 mtrie:333056 hash:3523 + totals: mtrie:666112 hash:7046 all:673158 + + Mtrie Mheap Usage: total: 32.06M, used: 662.44K, free: 31.42M, trimmable: 31.09M + free chunks 3 free fastbin blks 0 + max total allocated 32.06M + no traced allocations + +this output shows two 'empty' (i.e. no added routes) tables. Each +mtrie uses about 150k of memory, so each table about 300k. the total +heap usage statistics for the IP4 heap are shown at the end. + + +Below the output having added 1M, 2M and 4M routes respectively: + +.. code-block:: console + + vpp# sh ip fib mem + ipv4-VRF:0 mtrie:335744 hash:4695 + totals: mtrie:335744 hash:4695 all:340439 + + Mtrie Mheap Usage: total: 1.00G, used: 335.20K, free: 1023.74M, trimmable: 1023.72M + free chunks 3 free fastbin blks 0 + max total allocated 1.00G + no traced allocations + +.. code-block:: console + + vpp# sh ip fib mem + ipv4-VRF:0 mtrie:5414720 hash:41177579 + totals: mtrie:5414720 hash:41177579 all:46592299 + + Mtrie Mheap Usage: total: 1.00G, used: 46.87M, free: 977.19M, trimmable: 955.93M + free chunks 61 free fastbin blks 0 + max total allocated 1.00G + no traced allocations + +.. code-block:: console + + vpp# sh ip fib mem + ipv4-VRF:0 mtrie:22452608 hash:168544508 + totals: mtrie:22452608 hash:168544508 all:190997116 + + Mtrie Mheap Usage: total: 1.00G, used: 198.37M, free: 825.69M, trimmable: 748.24M + free chunks 219 free fastbin blks 0 + max total allocated 1.00G + no traced allocations + +VPP was started with a 1G IP4 heap. + +IP6 Heap +-------- + +The IPv6 heap is used to allocate the memory needed for the +data-structure within which the IPv6 prefixes are stored. IPv6 also +has the concept of forwarding and non-forwarding entries, however for +IPv6 all the forwardind entries are stored in a single hash table +(same goes for the non-forwarding). The key to the hash table includes +the IPv6 table-id. + +To see the amount of memory consumed by the IPv4 tables use: + +.. code-block:: console + + vpp# sh ip6 fib mem + IPv6 Non-Forwarding Hash Table: + Hash table ip6 FIB non-fwding table + 7 active elements 7 active buckets + 1 free lists + 0 linear search buckets + arena: base 7f2fe28bf000, next 803c0 + used 525248 b (0 Mbytes) of 33554432 b (32 Mbytes) + + IPv6 Forwarding Hash Table: + Hash table ip6 FIB fwding table + 7 active elements 7 active buckets + 1 free lists + 0 linear search buckets + arena: base 7f2fe48bf000, next 803c0 + used 525248 b (0 Mbytes) of 33554432 b (32 Mbytes) + +as we scale to 128k IPv6 entries: + +.. code-block:: console + + vpp# sh ip6 fib mem + IPv6 Non-Forwarding Hash Table: + Hash table ip6 FIB non-fwding table + 131079 active elements 32773 active buckets + 2 free lists + [len 1] 2 free elts + 0 linear search buckets + arena: base 7fed7a514000, next 4805c0 + used 4720064 b (4 Mbytes) of 1073741824 b (1024 Mbytes) + + IPv6 Forwarding Hash Table: + Hash table ip6 FIB fwding table + 131079 active elements 32773 active buckets + 2 free lists + [len 1] 2 free elts + 0 linear search buckets + arena: base 7fedba514000, next 4805c0 + used 4720064 b (4 Mbytes) of 1073741824 b (1024 Mbytes) + +and 256k: + +.. code-block:: console + + vpp# sh ip6 fib mem + IPv6 Non-Forwarding Hash Table: + Hash table ip6 FIB non-fwding table + 262151 active elements 65536 active buckets + 2 free lists + [len 1] 6 free elts + 0 linear search buckets + arena: base 7fed7a514000, next 880840 + used 8915008 b (8 Mbytes) of 1073741824 b (1024 Mbytes) + + IPv6 Forwarding Hash Table: + Hash table ip6 FIB fwding table + 262151 active elements 65536 active buckets + 2 free lists + [len 1] 6 free elts + 0 linear search buckets + arena: base 7fedba514000, next 880840 + used 8915008 b (8 Mbytes) of 1073741824 b (1024 Mbytes) + +and 1M: + +.. code-block:: console + + vpp# sh ip6 fib mem + IPv6 Non-Forwarding Hash Table: + Hash table ip6 FIB non-fwding table + 1048583 active elements 65536 active buckets + 4 free lists + [len 1] 65533 free elts + [len 2] 65531 free elts + [len 4] 9 free elts + 0 linear search buckets + arena: base 7fed7a514000, next 3882740 + used 59254592 b (56 Mbytes) of 1073741824 b (1024 Mbytes) + + IPv6 Forwarding Hash Table: + Hash table ip6 FIB fwding table + 1048583 active elements 65536 active buckets + 4 free lists + [len 1] 65533 free elts + [len 2] 65531 free elts + [len 4] 9 free elts + 0 linear search buckets + arena: base 7fedba514000, next 3882740 + used 59254592 b (56 Mbytes) of 1073741824 b (1024 Mbytes) + +as can be seen from the output the IPv6 heap in this case was scaled +to 1GB and 1million prefixes has used 56MB of it. + + +Main Heap +--------- + +The main heap is used to allocate objects that represent the FIB +entries in the control and data plane (see :ref:`controlplane` and +:ref:`dataplane`) such as *fib_entry_t* and *load_balance_t*. These come +from the main heap because they are not protocol specific +(i.e. they are used to represent either IPv4, IPv6 or MPLS +entries). + +With 1M prefixes allocated the memory usage is: + +.. code-block:: console + + vpp# sh fib mem + FIB memory + Tables: + SAFI Number Bytes + IPv4 unicast 1 33619968 + IPv6 unicast 2 118502784 + MPLS 0 0 + IPv4 multicast 1 1175 + IPv6 multicast 1 525312 + Nodes: + Name Size in-use /allocated totals + Entry 72 1048589/ 1048589 75498408/75498408 + Entry Source 40 1048589/ 1048589 41943560/41943560 + Entry Path-Extensions 76 0 / 0 0/0 + multicast-Entry 192 6 / 6 1152/1152 + Path-list 40 18 / 18 720/720 + uRPF-list 16 14 / 14 224/224 + Path 72 22 / 22 1584/1584 + Node-list elements 20 1048602/ 1048602 20972040/20972040 + Node-list heads 8 24 / 24 192/192 + +and with 2M + +.. code-block:: console + + vpp# sh fib mem + FIB memory + Tables: + SAFI Number Bytes + IPv4 unicast 1 33619968 + IPv6 unicast 2 252743040 + MPLS 0 0 + IPv4 multicast 1 1175 + IPv6 multicast 1 525312 + Nodes: + Name Size in-use /allocated totals + Entry 72 2097165/ 2097165 150995880/150995880 + Entry Source 40 2097165/ 2097165 83886600/83886600 + Entry Path-Extensions 76 0 / 0 0/0 + multicast-Entry 192 6 / 6 1152/1152 + Path-list 40 18 / 19 720/760 + uRPF-list 16 18 / 18 288/288 + Path 72 22 / 23 1584/1656 + Node-list elements 20 2097178/ 2097178 41943560/41943560 + Node-list heads 8 24 / 24 192/192 + +However, the situation is not a simple as that. All of the 1M prefixes +added above were reachable via the same next-hop, so the path-list +(and path) they use is shared. As prefixes are added that use +different (sets of) next-hops, the number of path-lists and paths +requires will increase. + + +Stats Heap +---------- + +VPP collects statistics for each route. For each route VPP collects +byte and packet counters for packets sent to the prefix (i.e. the +route was matched in the data-plane) and packets sent via the prefix (i.e. the +matching prefix is reachable through it - like a BGP peer). This +requires 4 counters per route in the stats segment. + +Below shows the size of the stats segment with 1M, 2M and 4M routes. + +.. code-block:: console + + total: 1023.99M, used: 127.89M, free: 896.10M, trimmable: 830.94M + total: 1023.99M, used: 234.14M, free: 789.85M, trimmable: 668.15M + total: 1023.99M, used: 456.83M, free: 567.17M, trimmable: 388.91M + +VPP was started with a 1G stats heap. + diff --git a/src/vnet/fib/fib_node.c b/src/vnet/fib/fib_node.c index d2e3f04de27..3e99007da64 100644 --- a/src/vnet/fib/fib_node.c +++ b/src/vnet/fib/fib_node.c @@ -237,7 +237,7 @@ fib_memory_show (vlib_main_t * vm, vlib_cli_output (vm, "FIB memory"); vlib_cli_output (vm, " Tables:"); - vlib_cli_output (vm, "%=30s %=6s %=8s", "SAFI", "Number", "Bytes"); + vlib_cli_output (vm, "%=30s %=6s %=12s", "SAFI", "Number", "Bytes"); vlib_cli_output (vm, "%U", format_fib_table_memory); vlib_cli_output (vm, "%U", format_mfib_table_memory); vlib_cli_output (vm, " Nodes:"); diff --git a/src/vnet/fib/ip4_fib.c b/src/vnet/fib/ip4_fib.c index 57fc23b6b19..8944637c127 100644 --- a/src/vnet/fib/ip4_fib.c +++ b/src/vnet/fib/ip4_fib.c @@ -566,12 +566,12 @@ u8 * format_ip4_fib_table_memory (u8 * s, va_list * args) { #if USE_DLMALLOC == 0 - s = format(s, "%=30s %=6d %=8ld\n", + s = format(s, "%=30s %=6d %=12ld\n", "IPv4 unicast", pool_elts(ip4_main.fibs), mheap_bytes(ip4_main.mtrie_mheap)); #else - s = format(s, "%=30s %=6d %=8ld\n", + s = format(s, "%=30s %=6d %=12ld\n", "IPv4 unicast", pool_elts(ip4_main.fibs), mspace_footprint(ip4_main.mtrie_mheap)); diff --git a/src/vnet/fib/ip6_fib.c b/src/vnet/fib/ip6_fib.c index 60d13659925..ba90c5ff1d3 100644 --- a/src/vnet/fib/ip6_fib.c +++ b/src/vnet/fib/ip6_fib.c @@ -573,16 +573,10 @@ format_ip6_fib_table_memory (u8 * s, va_list * args) { uword bytes_inuse; - bytes_inuse = - alloc_arena_next - (&(ip6_main.ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash)) - - alloc_arena (&(ip6_main.ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash)); + bytes_inuse = (alloc_arena_next(&(ip6_main.ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash)) + + alloc_arena_next(&(ip6_main.ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash))); - bytes_inuse += - alloc_arena_next(&(ip6_main.ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash)) - - alloc_arena(&(ip6_main.ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash)); - - s = format(s, "%=30s %=6d %=8ld\n", + s = format(s, "%=30s %=6d %=12ld\n", "IPv6 unicast", pool_elts(ip6_main.fibs), bytes_inuse); @@ -623,6 +617,7 @@ ip6_show_fib (vlib_main_t * vm, u32 mask_len = 128; int table_id = -1, fib_index = ~0; int detail = 0; + int hash = 0; verbose = 1; matching = 0; @@ -638,6 +633,11 @@ ip6_show_fib (vlib_main_t * vm, unformat (input, "det")) detail = 1; + else if (unformat (input, "hash") || + unformat (input, "mem") || + unformat (input, "memory")) + hash = 1; + else if (unformat (input, "%U/%d", unformat_ip6_address, &matching_address, &mask_len)) matching = 1; @@ -653,6 +653,19 @@ ip6_show_fib (vlib_main_t * vm, break; } + if (hash) + { + vlib_cli_output (vm, "IPv6 Non-Forwarding Hash Table:\n%U\n", + BV (format_bihash), + &im6->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash, + detail); + vlib_cli_output (vm, "IPv6 Forwarding Hash Table:\n%U\n", + BV (format_bihash), + &im6->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash, + detail); + return (NULL); + } + pool_foreach (fib_table, im6->fibs, ({ fib_source_t source; diff --git a/src/vnet/fib/mpls_fib.c b/src/vnet/fib/mpls_fib.c index 6670c99233f..6440689eab5 100644 --- a/src/vnet/fib/mpls_fib.c +++ b/src/vnet/fib/mpls_fib.c @@ -366,7 +366,7 @@ format_mpls_fib_table_memory (u8 * s, va_list * args) n_tables = pool_elts(mpls_main.fibs); mem = n_tables * sizeof(mpls_fib_t); - s = format(s, "%=30s %=6ld %=8ld\n", "MPLS", n_tables, mem); + s = format(s, "%=30s %=6ld %=12ld\n", "MPLS", n_tables, mem); return (s); } diff --git a/src/vnet/mfib/ip4_mfib.c b/src/vnet/mfib/ip4_mfib.c index 87603715d83..1d872f974fd 100644 --- a/src/vnet/mfib/ip4_mfib.c +++ b/src/vnet/mfib/ip4_mfib.c @@ -366,7 +366,7 @@ format_ip4_mfib_table_memory (u8 * s, va_list * args) total_memory += mfib_size; })); - s = format(s, "%=30s %=6d %=8ld\n", + s = format(s, "%=30s %=6d %=12ld\n", "IPv4 multicast", pool_elts(ip4_main.mfibs), total_memory); diff --git a/src/vnet/mfib/ip6_mfib.c b/src/vnet/mfib/ip6_mfib.c index aa9fdb9a98b..31a92687e1a 100644 --- a/src/vnet/mfib/ip6_mfib.c +++ b/src/vnet/mfib/ip6_mfib.c @@ -521,9 +521,14 @@ VLIB_INIT_FUNCTION(ip6_mfib_module_init); u8 * format_ip6_mfib_table_memory (u8 * s, va_list * args) { - s = format(s, "%=30s %=6d %=8s\n", + u64 bytes_inuse; + + bytes_inuse = alloc_arena_next(&(ip6_main.ip6_mtable.ip6_mhash)); + + s = format(s, "%=30s %=6d %=12ld\n", "IPv6 multicast", - pool_elts(ip6_main.mfibs), "???"); + pool_elts(ip6_main.mfibs), + bytes_inuse); return (s); } |