<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg version="1.2" width="163mm" height="127mm" viewBox="0 0 16300 12700" preserveAspectRatio="xMidYMid" fill-rule="evenodd" stroke-width="28.222" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" xmlns:ooo="http://xml.openoffice.org/svg/export" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:presentation="http://sun.com/xmlns/staroffice/presentation" xmlns:smil="http://www.w3.org/2001/SMIL20/" xmlns:anim="urn:oasis:names:tc:opendocument:xmlns:animation:1.0" xml:space="preserve">
<defs class="ClipPathGroup">
<clipPath id="presentation_clip_path" clipPathUnits="userSpaceOnUse">
<rect x="0" y="0" width="16300" height="12700"/>
</clipPath>
<clipPath id="presentation_clip_path_shrink" clipPathUnits="userSpaceOnUse">
<rect x="16" y="12" width="16268" height="12675"/>
</clipPath>
</defs>
<defs>
<font id="EmbeddedFont_1" horiz-adv-x="2048">
<font-face font-family="Liberation Sans embedded" units-per-em="2048" font-weight="normal" font-style="normal" ascent="1852" descent="423"/>
<missing-glyph horiz-adv-x="2048" d="M 0,0 L 2047,0 2047,2047 0,2047 0,0 Z"/>
<glyph unicode="x" horiz-adv-x="1006" d="M 801,0 L 510,444 217,0 23,0 408,556 41,1082 240,1082 510,661 778,1082 979,1082 612,558 1002,0 801,0 Z"/>
<glyph unicode="w" horiz-adv-x="1509" d="M 1174,0 L 965,0 776,765 740,934 C 734,904 725,861 712,805 699,748 631,480 508,0 L 300,0 -3,1082 175,1082 358,347 C 363,331 377,265 401,149 L 418,223 644,1082 837,1082 1026,339 1072,149 1103,288 1308,1082 1484,1082 1174,0 Z"/>
<glyph unicode="u" horiz-adv-x="874" d="M 314,1082 L 314,396 C 314,325 321,269 335,230 349,191 371,162 402,145 433,128 478,119 537,119 624,119 692,149 742,208 792,267 817,350 817,455 L 817,1082 997,1082 997,231 C 997,105 999,28 1003,0 L 833,0 C 832,3 832,12 831,27 830,42 830,59 829,78 828,97 826,132 825,185 L 822,185 C 781,110 733,58 679,27 624,-4 557,-20 476,-20 357,-20 271,10 216,69 161,128 133,225 133,361 L 133,1082 314,1082 Z"/>
<glyph unicode="t" horiz-adv-x="531" d="M 554,8 C 495,-8 434,-16 372,-16 228,-16 156,66 156,229 L 156,951 31,951 31,1082 163,1082 216,1324 336,1324 336,1082 536,1082 536,951 336,951 336,268 C 336,216 345,180 362,159 379,138 408,127 450,127 474,127 509,132 554,141 L 554,8 Z"/>
<glyph unicode="r" horiz-adv-x="530" d="M 142,0 L 142,830 C 142,906 140,990 136,1082 L 306,1082 C 311,959 314,886 314,861 L 318,861 C 347,954 380,1017 417,1051 454,1085 507,1102 575,1102 599,1102 623,1099 648,1092 L 648,927 C 624,934 592,937 552,937 477,937 420,905 381,841 342,776 322,684 322,564 L 322,0 142,0 Z"/>
<glyph unicode="q" horiz-adv-x="927" d="M 484,-20 C 347,-20 246,26 182,119 118,212 86,351 86,536 86,913 219,1102 484,1102 566,1102 634,1088 687,1059 740,1030 785,981 821,914 L 823,914 C 823,934 824,969 827,1018 830,1067 832,1093 835,1096 L 1008,1096 C 1003,1057 1001,958 1001,801 L 1001,-425 821,-425 821,14 825,178 823,178 C 787,107 743,56 690,26 637,-5 569,-20 484,-20 Z M 821,554 C 821,695 798,799 752,867 706,935 633,969 532,969 441,969 375,935 335,867 295,799 275,691 275,542 275,391 295,282 336,217 376,152 441,119 530,119 632,119 706,155 752,228 798,301 821,409 821,554 Z"/>
<glyph unicode="p" horiz-adv-x="953" d="M 1053,546 C 1053,169 920,-20 655,-20 488,-20 376,43 319,168 L 314,168 C 317,163 318,106 318,-2 L 318,-425 138,-425 138,861 C 138,972 136,1046 132,1082 L 306,1082 C 307,1079 308,1070 309,1054 310,1037 312,1012 314,978 315,944 316,921 316,908 L 320,908 C 352,975 394,1024 447,1055 500,1086 569,1101 655,1101 788,1101 888,1056 954,967 1020,878 1053,737 1053,546 Z M 864,542 C 864,693 844,800 803,865 762,930 698,962 609,962 538,962 482,947 442,917 401,887 371,840 350,777 329,713 318,630 318,528 318,386 341,281 386,214 431,147 505,113 607,113 696,113 762,146 803,212 844,277 864,387 864,542 Z"/>
<glyph unicode="o" horiz-adv-x="980" d="M 1053,542 C 1053,353 1011,212 928,119 845,26 724,-20 565,-20 407,-20 288,28 207,125 126,221 86,360 86,542 86,915 248,1102 571,1102 736,1102 858,1057 936,966 1014,875 1053,733 1053,542 Z M 864,542 C 864,691 842,800 798,868 753,935 679,969 574,969 469,969 393,935 346,866 299,797 275,689 275,542 275,399 298,292 345,221 391,149 464,113 563,113 671,113 748,148 795,217 841,286 864,395 864,542 Z"/>
<glyph unicode="n" horiz-adv-x="874" d="M 825,0 L 825,686 C 825,757 818,813 804,852 790,891 768,920 737,937 706,954 661,963 602,963 515,963 447,933 397,874 347,815 322,732 322,627 L 322,0 142,0 142,851 C 142,977 140,1054 136,1082 L 306,1082 C 307,1079 307,1070 308,1055 309,1040 310,1024 311,1005 312,986 313,950 314,897 L 317,897 C 358,972 406,1025 461,1056 515,1087 582,1102 663,1102 782,1102 869,1073 924,1014 979,955 1006,857 1006,721 L 1006,0 825,0 Z"/>
<glyph unicode="m" horiz-adv-x="1457" d="M 768,0 L 768,686 C 768,791 754,863 725,903 696,943 645,963 570,963 493,963 433,934 388,875 343,816 321,734 321,627 L 321,0 142,0 142,851 C 142,977 140,1054 136,1082 L 306,1082 C 307,1079 307,1070 308,1055 309,1040 310,1024 311,1005 312,986 313,950 314,897 L 317,897 C 356,974 400,1027 450,1057 500,1087 561,1102 633,1102 715,1102 780,1086 828,1053 875,1020 908,968 927,897 L 930,897 C 967,970 1013,1022 1066,1054 1119,1086 1183,1102 1258,1102 1367,1102 1447,1072 1497,1013 1546,954 1571,856 1571,721 L 1571,0 1393,0 1393,686 C 1393,791 1379,863 1350,903 1321,943 1270,963 1195,963 1116,963 1055,934 1012,876 968,817 946,734 946,627 L 946,0 768,0 Z"/>
<glyph unicode="k" horiz-adv-x="901" d="M 816,0 L 450,494 318,385 318,0 138,0 138,1484 318,1484 318,557 793,1082 1004,1082 565,617 1027,0 816,0 Z"/>
<glyph unicode="i" horiz-adv-x="187" d="M 137,1312 L 137,1484 317,1484 317,1312 137,1312 Z M 137,0 L 137,1082 317,1082 317,0 137,0 Z"/>
<glyph unicode="g" horiz-adv-x="927" d="M 548,-425 C 430,-425 336,-402 266,-356 196,-309 151,-243 131,-158 L 312,-132 C 324,-182 351,-220 392,-248 433,-274 486,-288 553,-288 732,-288 822,-183 822,27 L 822,201 820,201 C 786,132 739,80 680,45 621,10 551,-8 472,-8 339,-8 242,36 180,124 117,212 86,350 86,539 86,730 120,872 187,963 254,1054 355,1099 492,1099 569,1099 635,1082 692,1047 748,1012 791,962 822,897 L 824,897 C 824,917 825,952 828,1001 831,1050 833,1077 836,1082 L 1007,1082 C 1003,1046 1001,971 1001,858 L 1001,31 C 1001,-273 850,-425 548,-425 Z M 822,541 C 822,629 810,705 786,769 762,832 728,881 685,915 641,948 591,965 536,965 444,965 377,932 335,865 293,798 272,690 272,541 272,393 292,287 331,222 370,157 438,125 533,125 590,125 640,142 684,175 728,208 762,256 786,319 810,381 822,455 822,541 Z"/>
<glyph unicode="e" horiz-adv-x="980" d="M 276,503 C 276,379 302,283 353,216 404,149 479,115 578,115 656,115 719,131 766,162 813,193 844,233 861,281 L 1019,236 C 954,65 807,-20 578,-20 418,-20 296,28 213,123 129,218 87,360 87,548 87,727 129,864 213,959 296,1054 416,1102 571,1102 889,1102 1048,910 1048,527 L 1048,503 276,503 Z M 862,641 C 852,755 823,838 775,891 727,943 658,969 568,969 481,969 412,940 361,882 310,823 282,743 278,641 L 862,641 Z"/>
<glyph unicode="d" horiz-adv-x="927" d="M 821,174 C 788,105 744,55 689,25 634,-5 565,-20 484,-20 347,-20 247,26 183,118 118,210 86,349 86,536 86,913 219,1102 484,1102 566,1102 634,1087 689,1057 744,1027 788,979 821,914 L 823,914 821,1035 821,1484 1001,1484 1001,223 C 1001,110 1003,36 1007,0 L 835,0 C 833,11 831,35 829,74 826,113 825,146 825,174 L 821,174 Z M 275,542 C 275,391 295,282 335,217 375,152 440,119 530,119 632,119 706,154 752,225 798,296 821,405 821,554 821,697 798,802 752,869 706,936 633,969 532,969 441,969 376,936 336,869 295,802 275,693 275,542 Z"/>
<glyph unicode="c" horiz-adv-x="901" d="M 275,546 C 275,402 298,295 343,226 388,157 457,122 548,122 612,122 666,139 709,174 752,209 778,262 788,334 L 970,322 C 956,218 912,135 837,73 762,11 668,-20 553,-20 402,-20 286,28 207,124 127,219 87,359 87,542 87,724 127,863 207,959 287,1054 402,1102 551,1102 662,1102 754,1073 827,1016 900,959 945,880 964,779 L 779,765 C 770,825 746,873 708,908 670,943 616,961 546,961 451,961 382,929 339,866 296,803 275,696 275,546 Z"/>
<glyph unicode="b" horiz-adv-x="953" d="M 1053,546 C 1053,169 920,-20 655,-20 573,-20 505,-5 451,25 396,54 352,102 318,168 L 316,168 C 316,147 315,116 312,74 309,31 307,7 306,0 L 132,0 C 136,36 138,110 138,223 L 138,1484 318,1484 318,1061 C 318,1018 317,967 314,908 L 318,908 C 351,977 396,1027 451,1057 506,1087 574,1102 655,1102 792,1102 892,1056 957,964 1021,872 1053,733 1053,546 Z M 864,540 C 864,691 844,800 804,865 764,930 699,963 609,963 508,963 434,928 388,859 341,790 318,680 318,529 318,387 341,282 386,215 431,147 505,113 607,113 698,113 763,147 804,214 844,281 864,389 864,540 Z"/>
<glyph unicode="a" horiz-adv-x="1060" d="M 414,-20 C 305,-20 224,9 169,66 114,123 87,202 87,302 87,414 124,500 198,560 271,620 390,652 554,656 L 797,660 797,719 C 797,807 778,870 741,908 704,946 645,965 565,965 484,965 426,951 389,924 352,897 330,853 323,793 L 135,810 C 166,1005 310,1102 569,1102 705,1102 807,1071 876,1009 945,946 979,856 979,738 L 979,272 C 979,219 986,179 1000,152 1014,125 1041,111 1080,111 1097,111 1117,113 1139,118 L 1139,6 C 1094,-5 1047,-10 1000,-10 933,-10 885,8 855,43 824,78 807,132 803,207 L 797,207 C 751,124 698,66 637,32 576,-3 501,-20 414,-20 Z M 455,115 C 521,115 580,130 631,160 682,190 723,231 753,284 782,336 797,390 797,445 L 797,534 600,530 C 515,529 451,520 408,504 364,488 330,463 307,430 284,397 272,353 272,299 272,240 288,195 320,163 351,131 396,115 455,115 Z"/>
<glyph unicode="U" horiz-adv-x="1192" d="M 731,-20 C 616,-20 515,1 429,43 343,85 276,146 229,226 182,306 158,401 158,512 L 158,1409 349,1409 349,528 C 349,399 382,302 447,235 512,168 607,135 730,135 857,135 955,170 1026,239 1096,308 1131,408 1131,541 L 1131,1409 1321,1409 1321,530 C 1321,416 1297,318 1249,235 1200,152 1132,89 1044,46 955,2 851,-20 731,-20 Z"/>
<glyph unicode="S" horiz-adv-x="1192" d="M 1272,389 C 1272,259 1221,158 1120,87 1018,16 875,-20 690,-20 347,-20 148,99 93,338 L 278,375 C 299,290 345,228 414,189 483,149 578,129 697,129 820,129 916,150 983,193 1050,235 1083,297 1083,379 1083,425 1073,462 1052,491 1031,520 1001,543 963,562 925,581 880,596 827,609 774,622 716,635 652,650 541,675 456,699 399,724 341,749 295,776 262,807 229,837 203,872 186,913 168,954 159,1000 159,1053 159,1174 205,1267 298,1332 390,1397 522,1430 694,1430 854,1430 976,1406 1061,1357 1146,1308 1205,1224 1239,1106 L 1051,1073 C 1030,1148 991,1202 933,1236 875,1269 795,1286 692,1286 579,1286 493,1267 434,1230 375,1193 345,1137 345,1063 345,1020 357,984 380,956 403,927 436,903 479,884 522,864 609,840 738,811 781,801 825,791 868,781 911,770 952,758 991,744 1030,729 1067,712 1102,693 1136,674 1166,650 1191,622 1216,594 1236,561 1251,523 1265,485 1272,440 1272,389 Z"/>
<glyph unicode="N" horiz-adv-x="1165" d="M 1082,0 L 328,1200 333,1103 338,936 338,0 168,0 168,1409 390,1409 1152,201 C 1144,332 1140,426 1140,485 L 1140,1409 1312,1409 1312,0 1082,0 Z"/>
<glyph unicode="H" horiz-adv-x="1165" d="M 1121,0 L 1121,653 359,653 359,0 168,0 168,1409 359,1409 359,813 1121,813 1121,1409 1312,1409 1312,0 1121,0 Z"/>
<glyph unicode="1" horiz-adv-x="927" d="M 156,0 L 156,153 515,153 515,1237 197,1010 197,1180 530,1409 696,1409 696,153 1039,153 1039,0 156,0 Z"/>
<glyph unicode=" " horiz-adv-x="556"/>
</font>
</defs>
<defs>
<font id="EmbeddedFont_2" horiz-adv-x="2048">
<font-face font-family="Liberation Sans embedded" units-per-em="2048" font-weight="bold" font-style="normal" ascent="1852" descent="423"/>
<missing-glyph horiz-adv-x="2048" d="M 0,0 L 2047,0 2047,2047 0,2047 0,0 Z"/>
<glyph unicode="y" horiz-adv-x="1139" d="M 283,-425 C 216,-425 157,-421 106,-412 L 106,-212 C 141,-217 174,-220 203,-220 243,-220 276,-214 303,-201 329,-188 353,-167 374,-138 395,-109 418,-59 444,11 L 16,1082 313,1082 483,575 C 510,502 543,391 584,241 L 609,336 674,571 834,1082 1128,1082 700,-57 C 643,-196 583,-292 522,-345 460,-398 380,-425 283,-425 Z"/>
<glyph unicode="v" horiz-adv-x="1139" d="M 731,0 L 395,0 8,1082 305,1082 494,477 C 504,444 528,360 565,227 572,254 585,302 606,371 627,440 703,677 836,1082 L 1130,1082 731,0 Z"/>
<glyph unicode="t" horiz-adv-x="662" d="M 420,-18 C 337,-18 274,5 229,50 184,95 162,163 162,254 L 162,892 25,892 25,1082 176,1082 264,1336 440,1336 440,1082 645,1082 645,892 440,892 440,330 C 440,277 450,239 470,214 490,189 521,176 563,176 585,176 616,181 657,190 L 657,16 C 588,-7 509,-18 420,-18 Z"/>
<glyph unicode="s" horiz-adv-x="1006" d="M 1055,316 C 1055,211 1012,129 927,70 841,10 722,-20 571,-20 422,-20 309,4 230,51 151,98 98,171 72,270 L 319,307 C 333,256 357,219 392,198 426,177 486,166 571,166 650,166 707,176 743,196 779,216 797,247 797,290 797,325 783,352 754,373 725,393 675,410 606,424 447,455 340,485 285,512 230,539 188,574 159,617 130,660 115,712 115,775 115,878 155,959 235,1017 314,1074 427,1103 573,1103 702,1103 805,1078 884,1028 962,978 1011,906 1030,811 L 781,785 C 773,829 753,862 722,884 691,905 641,916 573,916 506,916 456,908 423,891 390,874 373,845 373,805 373,774 386,749 412,731 437,712 480,697 541,685 626,668 701,650 767,632 832,613 885,591 925,566 964,541 996,508 1020,469 1043,429 1055,378 1055,316 Z"/>
<glyph unicode="r" horiz-adv-x="636" d="M 143,0 L 143,828 C 143,887 142,937 141,977 139,1016 137,1051 135,1082 L 403,1082 C 405,1070 408,1034 411,973 414,912 416,871 416,851 L 420,851 C 447,927 472,981 493,1012 514,1043 540,1066 569,1081 598,1096 635,1103 679,1103 715,1103 744,1098 766,1088 L 766,853 C 721,863 681,868 646,868 576,868 522,840 483,783 444,726 424,642 424,531 L 424,0 143,0 Z"/>
<glyph unicode="o" horiz-adv-x="1113" d="M 1171,542 C 1171,367 1122,229 1025,130 928,30 793,-20 621,-20 452,-20 320,30 224,130 128,230 80,367 80,542 80,716 128,853 224,953 320,1052 454,1102 627,1102 804,1102 939,1054 1032,958 1125,861 1171,723 1171,542 Z M 877,542 C 877,671 856,764 814,822 772,880 711,909 631,909 460,909 375,787 375,542 375,421 396,330 438,267 479,204 539,172 618,172 791,172 877,295 877,542 Z"/>
<glyph unicode="n" horiz-adv-x="1007" d="M 844,0 L 844,607 C 844,797 780,892 651,892 583,892 528,863 487,805 445,746 424,671 424,580 L 424,0 143,0 143,840 C 143,898 142,946 141,983 139,1020 137,1053 135,1082 L 403,1082 C 405,1069 408,1036 411,981 414,926 416,888 416,867 L 420,867 C 458,950 506,1010 563,1047 620,1084 689,1103 768,1103 883,1103 971,1068 1032,997 1093,926 1124,823 1124,687 L 1124,0 844,0 Z"/>
<glyph unicode="m" horiz-adv-x="1562" d="M 780,0 L 780,607 C 780,797 725,892 616,892 559,892 513,863 478,805 442,747 424,672 424,580 L 424,0 143,0 143,840 C 143,898 142,946 141,983 139,1020 137,1053 135,1082 L 403,1082 C 405,1069 408,1036 411,981 414,926 416,888 416,867 L 420,867 C 455,950 498,1010 550,1047 601,1084 663,1103 735,1103 900,1103 1001,1024 1036,867 L 1042,867 C 1079,951 1123,1011 1174,1048 1225,1085 1291,1103 1370,1103 1475,1103 1556,1067 1611,996 1666,924 1694,821 1694,687 L 1694,0 1415,0 1415,607 C 1415,797 1360,892 1251,892 1196,892 1152,866 1117,813 1082,760 1062,686 1059,593 L 1059,0 780,0 Z"/>
<glyph unicode="l" horiz-adv-x="292" d="M 143,0 L 143,1484 424,1484 424,0 143,0 Z"/>
<glyph unicode="k" horiz-adv-x="1007" d="M 834,0 L 545,490 424,406 424,0 143,0 143,1484 424,1484 424,634 810,1082 1112,1082 732,660 1141,0 834,0 Z"/>
<glyph unicode="i" horiz-adv-x="292" d="M 143,1277 L 143,1484 424,1484 424,1277 143,1277 Z M 143,0 L 143,1082 424,1082 424,0 143,0 Z"/>
<glyph unicode="h" horiz-adv-x="1007" d="M 420,866 C 458,949 506,1009 563,1046 620,1083 689,1102 768,1102 883,1102 971,1067 1032,996 1093,925 1124,822 1124,686 L 1124,0 844,0 844,606 C 844,796 780,891 651,891 583,891 528,862 487,804 445,745 424,670 424,579 L 424,0 143,0 143,1484 424,1484 424,1079 C 424,1006 421,935 416,866 L 420,866 Z"/>
<glyph unicode="e" horiz-adv-x="1007" d="M 586,-20 C 423,-20 298,28 211,125 124,221 80,361 80,546 80,725 124,862 213,958 302,1054 427,1102 590,1102 745,1102 864,1051 946,948 1028,845 1069,694 1069,495 L 1069,487 375,487 C 375,382 395,302 434,249 473,195 528,168 600,168 699,168 762,211 788,297 L 1053,274 C 976,78 821,-20 586,-20 Z M 586,925 C 520,925 469,902 434,856 398,810 379,746 377,663 L 797,663 C 792,750 771,816 734,860 697,903 648,925 586,925 Z"/>
<glyph unicode="d" horiz-adv-x="1033" d="M 844,0 C 841,10 838,35 835,76 831,116 829,149 829,176 L 825,176 C 764,45 649,-20 479,-20 353,-20 256,29 187,128 118,226 84,363 84,540 84,719 120,858 193,956 265,1053 367,1102 500,1102 577,1102 643,1086 699,1054 754,1022 797,974 827,911 L 829,911 827,1089 827,1484 1108,1484 1108,236 C 1108,169 1111,91 1116,0 L 844,0 Z M 831,547 C 831,664 812,754 773,817 734,880 676,911 600,911 525,911 469,881 432,820 395,759 377,665 377,540 377,295 451,172 598,172 672,172 729,205 770,270 811,335 831,427 831,547 Z"/>
<glyph unicode="a" horiz-adv-x="1112" d="M 393,-20 C 288,-20 207,9 148,66 89,123 60,203 60,306 60,418 97,503 170,562 243,621 348,651 487,652 L 720,656 720,711 C 720,782 708,834 683,869 658,903 618,920 562,920 510,920 472,908 448,885 423,861 408,822 402,767 L 109,781 C 127,886 175,966 254,1021 332,1075 439,1102 574,1102 711,1102 816,1068 890,1001 964,934 1001,838 1001,714 L 1001,320 C 1001,259 1008,218 1022,195 1035,172 1058,160 1090,160 1111,160 1132,162 1152,166 L 1152,14 C 1135,10 1120,6 1107,3 1094,0 1080,-3 1067,-5 1054,-7 1040,-9 1025,-10 1010,-11 992,-12 972,-12 901,-12 849,5 816,40 782,75 762,126 755,193 L 749,193 C 670,51 552,-20 393,-20 Z M 720,501 L 576,499 C 511,496 464,489 437,478 410,466 389,448 375,424 360,400 353,368 353,328 353,277 365,239 389,214 412,189 444,176 483,176 527,176 567,188 604,212 640,236 668,269 689,312 710,354 720,399 720,446 L 720,501 Z"/>
<glyph unicode="U" horiz-adv-x="1244" d="M 723,-20 C 529,-20 381,27 278,122 175,217 123,352 123,528 L 123,1409 418,1409 418,551 C 418,440 445,355 498,298 551,240 628,211 731,211 836,211 917,241 974,302 1031,362 1059,448 1059,561 L 1059,1409 1354,1409 1354,543 C 1354,364 1299,226 1189,128 1078,29 923,-20 723,-20 Z"/>
<glyph unicode="T" horiz-adv-x="1245" d="M 773,1181 L 773,0 478,0 478,1181 23,1181 23,1409 1229,1409 1229,1181 773,1181 Z"/>
<glyph unicode="S" horiz-adv-x="1244" d="M 1286,406 C 1286,268 1235,163 1133,90 1030,17 880,-20 682,-20 501,-20 360,12 257,76 154,140 88,237 59,367 L 344,414 C 363,339 401,285 457,252 513,218 591,201 690,201 896,201 999,264 999,389 999,429 987,462 964,488 940,514 907,536 864,553 821,570 738,591 616,616 511,641 437,661 396,676 355,691 317,708 284,729 251,749 222,773 199,802 176,831 158,864 145,903 132,942 125,986 125,1036 125,1163 173,1261 269,1329 364,1396 503,1430 686,1430 861,1430 992,1403 1080,1348 1167,1293 1224,1203 1249,1077 L 963,1038 C 948,1099 919,1144 874,1175 829,1206 764,1221 680,1221 501,1221 412,1165 412,1053 412,1016 422,986 441,963 460,940 488,920 525,904 562,887 638,867 752,842 887,813 984,787 1043,763 1101,738 1147,710 1181,678 1215,645 1241,607 1259,562 1277,517 1286,465 1286,406 Z"/>
<glyph unicode="N" horiz-adv-x="1218" d="M 995,0 L 381,1085 C 393,980 399,895 399,831 L 399,0 137,0 137,1409 474,1409 1097,315 C 1085,416 1079,507 1079,590 L 1079,1409 1341,1409 1341,0 995,0 Z"/>
<glyph unicode="J" horiz-adv-x="980" d="M 524,-20 C 378,-20 266,12 188,75 109,138 57,241 31,382 L 324,425 C 336,352 358,299 391,264 424,229 469,211 526,211 585,211 629,231 660,270 690,309 705,366 705,439 L 705,1178 424,1178 424,1409 999,1409 999,446 C 999,299 957,185 874,103 791,21 674,-20 524,-20 Z"/>
<glyph unicode="I" horiz-adv-x="319" d="M 137,0 L 137,1409 432,1409 432,0 137,0 Z"/>
<glyph unicode="G" horiz-adv-x="1404" d="M 806,211 C 883,211 957,222 1029,245 1101,267 1157,295 1196,330 L 1196,525 852,525 852,743 1466,743 1466,225 C 1391,148 1294,88 1175,45 1055,2 929,-20 798,-20 569,-20 392,44 269,171 146,298 84,478 84,711 84,943 146,1121 270,1245 394,1368 572,1430 805,1430 1136,1430 1346,1308 1436,1063 L 1164,981 C 1135,1052 1089,1106 1026,1143 963,1180 890,1198 805,1198 666,1198 561,1156 489,1072 417,988 381,868 381,711 381,552 418,429 493,342 567,255 671,211 806,211 Z"/>
<glyph unicode="C" horiz-adv-x="1351" d="M 795,212 C 973,212 1097,301 1166,480 L 1423,383 C 1368,247 1287,146 1180,80 1073,13 944,-20 795,-20 568,-20 393,44 270,173 146,301 84,480 84,711 84,942 144,1120 263,1244 382,1368 555,1430 782,1430 947,1430 1082,1397 1186,1331 1290,1264 1363,1167 1405,1038 L 1145,967 C 1123,1038 1080,1094 1016,1136 951,1177 875,1198 788,1198 655,1198 554,1157 485,1074 416,991 381,870 381,711 381,549 417,425 488,340 559,255 661,212 795,212 Z"/>
<glyph unicode="1" horiz-adv-x="980" d="M 129,0 L 129,209 478,209 478,1170 140,959 140,1180 493,1409 759,1409 759,209 1082,209 1082,0 129,0 Z"/>
<glyph unicode=" " horiz-adv-x="556"/>
</font>
</defs>
<defs class="TextShapeIndex">
<g ooo:slide="id1" ooo:id-list="id3 id4 id5 id6 id7 id8 id9 id10 id11 id12 id13 id14 id15 id16 id17 id18 id19 id20 id21 id22 id23 id24 id25 id26"/>
</defs>
<defs class="EmbeddedBulletChars">
<g id="bullet-char-template-57356" transform="scale(0.00048828125,-0.00048828125)">
<path d="M 580,1141 L 1163,571 580,0 -4,571 580,1141 Z"/>
</g>
<g id="bullet-char-template-57354" transform="scale(0.00048828125,-0.00048828125)">
<path d="M 8,1128 L 1137,1128 1137,0 8,0 8,1128 Z"/>
</g>
<g id="bullet-char-template-10146" transform="scale(0.00048828125,-0.00048828125)">
<path d="M 174,0 L 602,739 174,1481 1456,739 174,0 Z M 1358,739 L 309,1346 659,739 1358,739 Z"/>
</g>
<g id="bullet-char-template-10132" transform="scale(0.00048828125,-0.00048828125)">
<path d="M 2015,739 L 1276,0 717,0 1260,543 174,543 174,936 1260,936 717,1481 1274,1481 2015,739 Z"/>
</g>
<g id="bullet-char-template-10007" transform="scale(0.00048828125,-0.00048828125)">
<path d="M 0,-2 C -7,14 -16,27 -25,37 L 356,567 C 262,823 215,952 215,954 215,979 228,992 255,992 264,992 276,990 289,987 310,991 331,999 354,1012 L 381,999 492,748 772,1049 836,1024 860,1049 C 881,1039 901,1025 922,1006 886,937 835,863 770,784 769,783 710,716 594,584 L 774,223 C 774,196 753,168 711,139 L 727,119 C 717,90 699,76 672,76 641,76 570,178 457,381 L 164,-76 C 142,-110 111,-127 72,-127 30,-127 9,-110 8,-76 1,-67 -2,-52 -2,-32 -2,-23 -1,-13 0,-2 Z"/>
</g>
<g id="bullet-char-template-10004" transform="scale(0.00048828125,-0.00048828125)">
<path d="M 285,-33 C 182,-33 111,30 74,156 52,228 41,333 41,471 41,549 55,616 82,672 116,743 169,778 240,778 293,778 328,747 346,684 L 369,508 C 377,444 397,411 428,410 L 1163,1116 C 1174,1127 1196,1133 1229,1133 1271,1133 1292,1118 1292,1087 L 1292,965 C 1292,929 1282,901 1262,881 L 442,47 C 390,-6 338,-33 285,-33 Z"/>
</g>
<g id="bullet-char-template-9679" transform="scale(0.00048828125,-0.00048828125)">
<path d="M 813,0 C 632,0 489,54 383,161 276,268 223,411 223,592 223,773 276,916 383,1023 489,1130 632,1184 813,1184 992,1184 1136,1130 1245,1023 1353,916 1407,772 1407,592 1407,412 1353,268 1245,161 1136,54 992,0 813,0 Z"/>
</g>
<g id="bullet-char-template-8226" transform="scale(0.00048828125,-0.00048828125)">
<path d="M 346,457 C 273,457 209,483 155,535 101,586 74,649 74,723 74,796 101,859 155,911 209,963 273,989 346,989 419,989 480,963 531,910 582,859 608,796 608,723 608,648 583,586 532,535 482,483 420,457 346,457 Z"/>
</g>
<g id="bullet-char-template-8211" transform="scale(0.00048828125,-0.00048828125)">
<path d="M -4,459 L 1135,459 1135,606 -4,606 -4,459 Z"/>
</g>
<g id="bullet-char-template-61548" transform="scale(0.00048828125,-0.00048828125)">
<path d="M 173,740 C 173,903 231,1043 346,1159 462,1274 601,1332 765,1332 928,1332 1067,1274 1183,1159 1299,1043 1357,903 1357,740 1357,577 1299,437 1183,322 1067,206 928,148 765,148 601,148 462,206 346,322 231,437 173,577 173,740 Z"/>
</g>
</defs>
<defs class="TextEmbeddedBitmaps"/>
<g>
<g id="id2" class="Master_Slide">
<g id="bg-id2" class="Background"/*
*------------------------------------------------------------------
* vhost-user-input
*
* Copyright (c) 2014-2018 Cisco and/or its affiliates.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*------------------------------------------------------------------
*/
#include <fcntl.h> /* for open */
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/uio.h> /* for iovec */
#include <netinet/in.h>
#include <sys/vfs.h>
#include <linux/if_arp.h>
#include <linux/if_tun.h>
#include <vlib/vlib.h>
#include <vlib/unix/unix.h>
#include <vnet/ip/ip.h>
#include <vnet/ethernet/ethernet.h>
#include <vnet/devices/devices.h>
#include <vnet/feature/feature.h>
#include <vnet/devices/virtio/virtio.h>
#include <vnet/devices/virtio/vhost_user.h>
#include <vnet/devices/virtio/vhost_user_inline.h>
/*
* When an RX queue is down but active, received packets
* must be discarded. This value controls up to how many
* packets will be discarded during each round.
*/
#define VHOST_USER_DOWN_DISCARD_COUNT 256
/*
* When the number of available buffers gets under this threshold,
* RX node will start discarding packets.
*/
#define VHOST_USER_RX_BUFFER_STARVATION 32
/*
* On the receive side, the host should free descriptors as soon
* as possible in order to avoid TX drop in the VM.
* This value controls the number of copy operations that are stacked
* before copy is done for all and descriptors are given back to
* the guest.
* The value 64 was obtained by testing (48 and 128 were not as good).
*/
#define VHOST_USER_RX_COPY_THRESHOLD 64
extern vlib_node_registration_t vhost_user_input_node;
#define foreach_vhost_user_input_func_error \
_(NO_ERROR, "no error") \
_(NO_BUFFER, "no available buffer") \
_(MMAP_FAIL, "mmap failure") \
_(INDIRECT_OVERFLOW, "indirect descriptor overflows table") \
_(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)") \
_(NOT_READY, "vhost interface not ready or down") \
_(FULL_RX_QUEUE, "full rx queue (possible driver tx drop)")
typedef enum
{
#define _(f,s) VHOST_USER_INPUT_FUNC_ERROR_##f,
foreach_vhost_user_input_func_error
#undef _
VHOST_USER_INPUT_FUNC_N_ERROR,
} vhost_user_input_func_error_t;
static __clib_unused char *vhost_user_input_func_error_strings[] = {
#define _(n,s) s,
foreach_vhost_user_input_func_error
#undef _
};
static_always_inline void
vhost_user_rx_trace (vhost_trace_t * t,
vhost_user_intf_t * vui, u16 qid,
vlib_buffer_t * b, vhost_user_vring_t * txvq,
u16 last_avail_idx)
{
vhost_user_main_t *vum = &vhost_user_main;
u32 desc_current = txvq->avail->ring[last_avail_idx & txvq->qsz_mask];
vring_desc_t *hdr_desc = 0;
virtio_net_hdr_mrg_rxbuf_t *hdr;
u32 hint = 0;
clib_memset (t, 0, sizeof (*t));
t->device_index = vui - vum->vhost_user_interfaces;
t->qid = qid;
hdr_desc = &txvq->desc[desc_current];
if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
{
t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT;
/* Header is the first here */
hdr_desc = map_guest_mem (vui, txvq->desc[desc_current].addr, &hint);
}
if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT)
{
t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED;
}
if (!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) &&
!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT))
{
t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC;
}
t->first_desc_len = hdr_desc ? hdr_desc->len : 0;
if (!hdr_desc || !(hdr = map_guest_mem (vui, hdr_desc->addr, &hint)))
{
t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_MAP_ERROR;
}
else
{
u32 len = vui->virtio_net_hdr_sz;
memcpy (&t->hdr, hdr, len > hdr_desc->len ? hdr_desc->len : len);
}
}
static_always_inline u32
vhost_user_input_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy,
u16 copy_len, u32 * map_hint)
{
void *src0, *src1, *src2, *src3;
if (PREDICT_TRUE (copy_len >= 4))
{
if (PREDICT_FALSE (!(src2 = map_guest_mem (vui, cpy[0].src, map_hint))))
return 1;
if (PREDICT_FALSE (!(src3 = map_guest_mem (vui, cpy[1].src, map_hint))))
return 1;
while (PREDICT_TRUE (copy_len >= 4))
{
src0 = src2;
src1 = src3;
if (PREDICT_FALSE
(!(src2 = map_guest_mem (vui, cpy[2].src, map_hint))))
return 1;
if (PREDICT_FALSE
(!(src3 = map_guest_mem (vui, cpy[3].src, map_hint))))
return 1;
CLIB_PREFETCH (src2, 64, LOAD);
CLIB_PREFETCH (src3, 64, LOAD);
clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len);
clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len);
copy_len -= 2;
cpy += 2;
}
}
while (copy_len)
{
if (PREDICT_FALSE (!(src0 = map_guest_mem (vui, cpy->src, map_hint))))
return 1;
clib_memcpy_fast ((void *) cpy->dst, src0, cpy->len);
copy_len -= 1;
cpy += 1;
}
return 0;
}
/**
* Try to discard packets from the tx ring (VPP RX path).
* Returns the number of discarded packets.
*/
static_always_inline u32
vhost_user_rx_discard_packet (vlib_main_t * vm,
vhost_user_intf_t * vui,
vhost_user_vring_t * txvq, u32 discard_max)
{
/*
* On the RX side, each packet corresponds to one descriptor
* (it is the same whether it is a shallow descriptor, chained, or indirect).
* Therefore, discarding a packet is like discarding a descriptor.
*/
u32 discarded_packets = 0;
u32 avail_idx = txvq->avail->idx;
u16 mask = txvq->qsz_mask;
u16 last_avail_idx = txvq->last_avail_idx;
u16 last_used_idx = txvq->last_used_idx;
while (discarded_packets != discard_max)
{
if (avail_idx == last_avail_idx)
goto out;
u16 desc_chain_head = txvq->avail->ring[last_avail_idx & mask];
last_avail_idx++;
txvq->used->ring[last_used_idx & mask].id = desc_chain_head;
txvq->used->ring[last_used_idx & mask].len = 0;
vhost_user_log_dirty_ring (vui, txvq, ring[last_used_idx & mask]);
last_used_idx++;
discarded_packets++;
}
out:
txvq->last_avail_idx = last_avail_idx;
txvq->last_used_idx = last_used_idx;
CLIB_MEMORY_STORE_BARRIER ();
txvq->used->idx = txvq->last_used_idx;
vhost_user_log_dirty_ring (vui, txvq, idx);
return discarded_packets;
}
/*
* In case of overflow, we need to rewind the array of allocated buffers.
*/
static_always_inline void
vhost_user_input_rewind_buffers (vlib_main_t * vm,
vhost_cpu_t * cpu, vlib_buffer_t * b_head)
{
u32 bi_current = cpu->rx_buffers[cpu->rx_buffers_len];
vlib_buffer_t *b_current = vlib_get_buffer (vm, bi_current);
b_current->current_length = 0;
b_current->flags = 0;
while (b_current != b_head)
{
cpu->rx_buffers_len++;
bi_current = cpu->rx_buffers[cpu->rx_buffers_len];
b_current = vlib_get_buffer (vm, bi_current);
b_current->current_length = 0;
b_current->flags = 0;
}
cpu->rx_buffers_len++;
}
static_always_inline void
vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
virtio_net_hdr_t * hdr)
{
u8 l4_hdr_sz = 0;
u8 l4_proto = 0;
ethernet_header_t *eh = (ethernet_header_t *) b0_data;
u16 ethertype = clib_net_to_host_u16 (eh->type);
u16 l2hdr_sz = sizeof (ethernet_header_t);
if (ethernet_frame_is_tagged (ethertype))
{
ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1);
ethertype = clib_net_to_host_u16 (vlan->type);
l2hdr_sz += sizeof (*vlan);
if (ethertype == ETHERNET_TYPE_VLAN)
{
vlan++;
ethertype = clib_net_to_host_u16 (vlan->type);
l2hdr_sz += sizeof (*vlan);
}
}
vnet_buffer (b0)->l2_hdr_offset = 0;
vnet_buffer (b0)->l3_hdr_offset = l2hdr_sz;
vnet_buffer (b0)->l4_hdr_offset = hdr->csum_start;
b0->flags |= (VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4))
{
ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz);
l4_proto = ip4->protocol;
b0->flags |= VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
{
ip6_header_t *ip6 = (ip6_header_t *) (b0_data + l2hdr_sz);
l4_proto = ip6->protocol;
b0->flags |= VNET_BUFFER_F_IS_IP6;
}
if (l4_proto == IP_PROTOCOL_TCP)
{
tcp_header_t *tcp = (tcp_header_t *)
(b0_data + vnet_buffer (b0)->l4_hdr_offset);
l4_hdr_sz = tcp_header_bytes (tcp);
tcp->checksum = 0;
b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
}
else if (l4_proto == IP_PROTOCOL_UDP)
{
udp_header_t *udp =
(udp_header_t *) (b0_data + vnet_buffer (b0)->l4_hdr_offset);
l4_hdr_sz = sizeof (*udp);
udp->checksum = 0;
b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
}
if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP)
{
vnet_buffer2 (b0)->gso_size = hdr->gso_size;
vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
b0->flags |= VNET_BUFFER_F_GSO;
}
else if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV4)
{
vnet_buffer2 (b0)->gso_size = hdr->gso_size;
vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4);
}
else if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV6)
{
vnet_buffer2 (b0)->gso_size = hdr->gso_size;
vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
}
}
static_always_inline void
vhost_user_input_do_interrupt (vlib_main_t * vm, vhost_user_vring_t * txvq,
vhost_user_vring_t * rxvq)
{
f64 now = vlib_time_now (vm);
if ((txvq->n_since_last_int) && (txvq->int_deadline < now))
vhost_user_send_call (vm, txvq);
if ((rxvq->n_since_last_int) && (rxvq->int_deadline < now))
vhost_user_send_call (vm, rxvq);
}
static_always_inline void
vhost_user_input_setup_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
vhost_user_intf_t * vui,
u32 * current_config_index, u32 * next_index,
u32 ** to_next, u32 * n_left_to_next)
{
vnet_feature_main_t *fm = &feature_main;
u8 feature_arc_idx = fm->device_input_feature_arc_index;
if (PREDICT_FALSE (vnet_have_features (feature_arc_idx, vui->sw_if_index)))
{
vnet_feature_config_main_t *cm;
cm = &fm->feature_config_mains[feature_arc_idx];
*current_config_index = vec_elt (cm->config_index_by_sw_if_index,
vui->sw_if_index);
vnet_get_config_data (&cm->config_main, current_config_index,
next_index, 0);
}
vlib_get_new_next_frame (vm, node, *next_index, *to_next, *n_left_to_next);
if (*next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
{
/* give some hints to ethernet-input */
vlib_next_frame_t *nf;
vlib_frame_t *f;
ethernet_input_frame_t *ef;
nf = vlib_node_runtime_get_next_frame (vm, node, *next_index);
f = vlib_get_frame (vm, nf->frame);
f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX;
ef = vlib_frame_scalar_args (f);
ef->sw_if_index = vui->sw_if_index;
ef->hw_if_index = vui->hw_if_index;
vlib_frame_no_append (f);
}
}
static_always_inline u32
vhost_user_if_input (vlib_main_t * vm,
vhost_user_main_t * vum,
vhost_user_intf_t * vui,
u16 qid, vlib_node_runtime_t * node,
vnet_hw_interface_rx_mode mode, u8 enable_csum)
{
vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
vnet_feature_main_t *fm = &feature_main;
u16 n_rx_packets = 0;
u32 n_rx_bytes = 0;
u16 n_left;
u32 n_left_to_next, *to_next;
u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
u32 n_trace = vlib_get_trace_count (vm, node);
u32 buffer_data_size = vlib_buffer_get_default_data_size (vm);
u32 map_hint = 0;
vhost_cpu_t *cpu = &vum->cpus[vm->thread_index];
u16 copy_len = 0;
u8 feature_arc_idx = fm->device_input_feature_arc_index;
u32 current_config_index = ~(u32) 0;
u16 mask = txvq->qsz_mask;
/* The descriptor table is not ready yet */
if (PREDICT_FALSE (txvq->avail == 0))
goto done;
{
/* do we have pending interrupts ? */
vhost_user_vring_t *rxvq = &vui->vrings[VHOST_VRING_IDX_RX (qid)];
vhost_user_input_do_interrupt (vm, txvq, rxvq);
}
/*
* For adaptive mode, it is optimized to reduce interrupts.
* If the scheduler switches the input node to polling due
* to burst of traffic, we tell the driver no interrupt.
* When the traffic subsides, the scheduler switches the node back to
* interrupt mode. We must tell the driver we want interrupt.
*/
if (PREDICT_FALSE (mode == VNET_HW_INTERFACE_RX_MODE_ADAPTIVE))
{
if ((node->flags &
VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE) ||
!(node->flags &
VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE))
/* Tell driver we want notification */
txvq->used->flags = 0;
else
/* Tell driver we don't want notification */
txvq->used->flags = VRING_USED_F_NO_NOTIFY;
}
if (PREDICT_FALSE (txvq->avail->flags & 0xFFFE))
goto done;
n_left = (u16) (txvq->avail->idx - txvq->last_avail_idx);
/* nothing to do */
if (PREDICT_FALSE (n_left == 0))
goto done;
if (PREDICT_FALSE (!vui->admin_up || !(txvq->enabled)))
{
/*
* Discard input packet if interface is admin down or vring is not
* enabled.
* "For example, for a networking device, in the disabled state
* client must not supply any new RX packets, but must process
* and discard any TX packets."
*/
vhost_user_rx_discard_packet (vm, vui, txvq,
VHOST_USER_DOWN_DISCARD_COUNT);
goto done;
}
if (PREDICT_FALSE (n_left == (mask + 1)))
{
/*
* Informational error logging when VPP is not
* receiving packets fast enough.
*/
vlib_error_count (vm, node->node_index,
VHOST_USER_INPUT_FUNC_ERROR_FULL_RX_QUEUE, 1);
}
if (n_left > VLIB_FRAME_SIZE)
n_left = VLIB_FRAME_SIZE;
/*
* For small packets (<2kB), we will not need more than one vlib buffer
* per packet. In case packets are bigger, we will just yield at some point
* in the loop and come back later. This is not an issue as for big packet,
* processing cost really comes from the memory copy.
* The assumption is that big packets will fit in 40 buffers.
*/
if (PREDICT_FALSE (cpu->rx_buffers_len < n_left + 1 ||
cpu->rx_buffers_len < 40))
{
u32 curr_len = cpu->rx_buffers_len;
cpu->rx_buffers_len +=
vlib_buffer_alloc (vm, cpu->rx_buffers + curr_len,
VHOST_USER_RX_BUFFERS_N - curr_len);
if (PREDICT_FALSE
(cpu->rx_buffers_len < VHOST_USER_RX_BUFFER_STARVATION))
{
/* In case of buffer starvation, discard some packets from the queue
* and log the event.
* We keep doing best effort for the remaining packets. */
u32 flush = (n_left + 1 > cpu->rx_buffers_len) ?
n_left + 1 - cpu->rx_buffers_len : 1;
flush = vhost_user_rx_discard_packet (vm, vui, txvq, flush);
n_left -= flush;
vlib_increment_simple_counter (vnet_main.
interface_main.sw_if_counters +
VNET_INTERFACE_COUNTER_DROP,
vm->thread_index, vui->sw_if_index,
flush);
vlib_error_count (vm, vhost_user_input_node.index,
VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, flush);
}
}
vhost_user_input_setup_frame (vm, node, vui, ¤t_config_index,
&next_index, &to_next, &n_left_to_next);
u16 last_avail_idx = txvq->last_avail_idx;
u16 last_used_idx = txvq->last_used_idx;
while (n_left > 0)
{
vlib_buffer_t *b_head, *b_current;
u32 bi_current;
u16 desc_current;
u32 desc_data_offset;
vring_desc_t *desc_table = txvq->desc;
if (PREDICT_FALSE (cpu->rx_buffers_len <= 1))
{
/* Not enough rx_buffers
* Note: We yeld on 1 so we don't need to do an additional
* check for the next buffer prefetch.
*/
n_left = 0;
break;
}
desc_current = txvq->avail->ring[last_avail_idx & mask];
cpu->rx_buffers_len--;
bi_current = cpu->rx_buffers[cpu->rx_buffers_len];
b_head = b_current = vlib_get_buffer (vm, bi_current);
to_next[0] = bi_current; //We do that now so we can forget about bi_current
to_next++;
n_left_to_next--;
vlib_prefetch_buffer_with_index
(vm, cpu->rx_buffers[cpu->rx_buffers_len - 1], LOAD);
/* Just preset the used descriptor id and length for later */
txvq->used->ring[last_used_idx & mask].id = desc_current;
txvq->used->ring[last_used_idx & mask].len = 0;
vhost_user_log_dirty_ring (vui, txvq, ring[last_used_idx & mask]);
/* The buffer should already be initialized */
b_head->total_length_not_including_first_buffer = 0;
b_head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
if (PREDICT_FALSE (n_trace))
{
vlib_trace_buffer (vm, node, next_index, b_head,
/* follow_chain */ 0);
vhost_trace_t *t0 =
vlib_add_trace (vm, node, b_head, sizeof (t0[0]));
vhost_user_rx_trace (t0, vui, qid, b_head, txvq, last_avail_idx);
n_trace--;
vlib_set_trace_count (vm, node, n_trace);
}
/* This depends on the setup but is very consistent
* So I think the CPU branch predictor will make a pretty good job
* at optimizing the decision. */
if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
{
desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr,
&map_hint);
desc_current = 0;
if (PREDICT_FALSE (desc_table == 0))
{
vlib_error_count (vm, node->node_index,
VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
goto out;
}
}
desc_data_offset = vui->virtio_net_hdr_sz;
if (enable_csum)
{
virtio_net_hdr_mrg_rxbuf_t *hdr;
u8 *b_data;
u16 current;
hdr = map_guest_mem (vui, desc_table[desc_current].addr, &map_hint);
if (PREDICT_FALSE (hdr == 0))
{
vlib_error_count (vm, node->node_index,
VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
goto out;
}
if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
{
if ((desc_data_offset == desc_table[desc_current].len) &&
(desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT))
{
current = desc_table[desc_current].next;
b_data = map_guest_mem (vui, desc_table[current].addr,
&map_hint);
if (PREDICT_FALSE (b_data == 0))
{
vlib_error_count (vm, node->node_index,
VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL,
1);
goto out;
}
}
else
b_data = (u8 *) hdr + desc_data_offset;
vhost_user_handle_rx_offload (b_head, b_data, &hdr->hdr);
}
}
while (1)
{
/* Get more input if necessary. Or end of packet. */
if (desc_data_offset == desc_table[desc_current].len)
{
if (PREDICT_FALSE (desc_table[desc_current].flags &
VIRTQ_DESC_F_NEXT))
{
desc_current = desc_table[desc_current].next;
desc_data_offset = 0;
}
else
{
goto out;
}
}
/* Get more output if necessary. Or end of packet. */
if (PREDICT_FALSE (b_current->current_length == buffer_data_size))
{
if (PREDICT_FALSE (cpu->rx_buffers_len == 0))
{
/* Cancel speculation */
to_next--;
n_left_to_next++;
/*
* Checking if there are some left buffers.
* If not, just rewind the used buffers and stop.
* Note: Scheduled copies are not cancelled. This is
* not an issue as they would still be valid. Useless,
* but valid.
*/
vhost_user_input_rewind_buffers (vm, cpu, b_head);
n_left = 0;
goto stop;
}
/* Get next output */
cpu->rx_buffers_len--;
u32 bi_next = cpu->rx_buffers[cpu->rx_buffers_len];
b_current->next_buffer = bi_next;
b_current->flags |= VLIB_BUFFER_NEXT_PRESENT;
bi_current = bi_next;
b_current = vlib_get_buffer (vm, bi_current);
}
/* Prepare a copy order executed later for the data */
ASSERT (copy_len < VHOST_USER_COPY_ARRAY_N);
vhost_copy_t *cpy = &cpu->copy[copy_len];
copy_len++;
u32 desc_data_l = desc_table[desc_current].len - desc_data_offset;
cpy->len = buffer_data_size - b_current->current_length;
cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len;
cpy->dst = (uword) (vlib_buffer_get_current (b_current) +
b_current->current_length);
cpy->src = desc_table[desc_current].addr + desc_data_offset;
desc_data_offset += cpy->len;
b_current->current_length += cpy->len;
b_head->total_length_not_including_first_buffer += cpy->len;
}
out:
n_rx_bytes += b_head->total_length_not_including_first_buffer;
n_rx_packets++;
b_head->total_length_not_including_first_buffer -=
b_head->current_length;
/* consume the descriptor and return it as used */
last_avail_idx++;
last_used_idx++;
VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b_head);
vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index;
vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32) ~ 0;
b_head->error = 0;
if (current_config_index != ~(u32) 0)
{
b_head->current_config_index = current_config_index;
vnet_buffer (b_head)->feature_arc_index = feature_arc_idx;
}
n_left--;
/*
* Although separating memory copies from virtio ring parsing
* is beneficial, we can offer to perform the copies from time
* to time in order to free some space in the ring.
*/
if (PREDICT_FALSE (copy_len >= VHOST_USER_RX_COPY_THRESHOLD))
{
if (PREDICT_FALSE (vhost_user_input_copy (vui, cpu->copy,
copy_len, &map_hint)))
{
vlib_error_count (vm, node->node_index,
VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
}
copy_len = 0;
/* give buffers back to driver */
CLIB_MEMORY_STORE_BARRIER ();
txvq->used->idx = last_used_idx;
vhost_user_log_dirty_ring (vui, txvq, idx);
}
}
stop:
vlib_put_next_frame (vm, node, next_index, n_left_to_next);
txvq->last_used_idx = last_used_idx;
txvq->last_avail_idx = last_avail_idx;
/* Do the memory copies */
if (PREDICT_FALSE (vhost_user_input_copy (vui, cpu->copy, copy_len,
&map_hint)))
{
vlib_error_count (vm, node->node_index,
VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
}
/* give buffers back to driver */
CLIB_MEMORY_STORE_BARRIER ();
txvq->used->idx = txvq->last_used_idx;
vhost_user_log_dirty_ring (vui, txvq, idx);
/* interrupt (call) handling */
if ((txvq->callfd_idx != ~0) &&
!(txvq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
{
txvq->n_since_last_int += n_rx_packets;
if (txvq->n_since_last_int > vum->coalesce_frames)
vhost_user_send_call (vm, txvq);
}
/* increase rx counters */
vlib_increment_combined_counter
(vnet_main.interface_main.combined_sw_if_counters
+ VNET_INTERFACE_COUNTER_RX, vm->thread_index, vui->sw_if_index,
n_rx_packets, n_rx_bytes);
vnet_device_increment_rx_packets (vm->thread_index, n_rx_packets);
done:
return n_rx_packets;
}
static_always_inline void
vhost_user_mark_desc_consumed (vhost_user_intf_t * vui,
vhost_user_vring_t * txvq, u16 desc_head,
u16 n_descs_processed)
{
vring_packed_desc_t *desc_table = txvq->packed_desc;
u16 desc_idx;
u16 mask = txvq->qsz_mask;
for (desc_idx = 0; desc_idx < n_descs_processed; desc_idx++)
{
if (txvq->used_wrap_counter)
desc_table[(desc_head + desc_idx) & mask].flags |=
(VIRTQ_DESC_F_AVAIL | VIRTQ_DESC_F_USED);
else
desc_table[(desc_head + desc_idx) & mask].flags &=
~(VIRTQ_DESC_F_AVAIL | VIRTQ_DESC_F_USED);
vhost_user_advance_last_used_idx (txvq);
}
}
static_always_inline void
vhost_user_rx_trace_packed (vhost_trace_t * t, vhost_user_intf_t * vui,
u16 qid, vhost_user_vring_t * txvq,
u16 desc_current)
{
vhost_user_main_t *vum = &vhost_user_main;
vring_packed_desc_t *hdr_desc;
virtio_net_hdr_mrg_rxbuf_t *hdr;
u32 hint = 0;
clib_memset (t, 0, sizeof (*t));
t->device_index = vui - vum->vhost_user_interfaces;
t->qid = qid;
hdr_desc = &txvq->packed_desc[desc_current];
if (txvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
{
t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT;
/* Header is the first here */
hdr_desc = map_guest_mem (vui, txvq->packed_desc[desc_current].addr,
&hint);
}
if (txvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_NEXT)
t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED;
if (!(txvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_NEXT) &&
!(txvq->packed_desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT))
t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC;
t->first_desc_len = hdr_desc ? hdr_desc->len : 0;
if (!hdr_desc || !(hdr = map_guest_mem (vui, hdr_desc->addr, &hint)))
t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_MAP_ERROR;
else
{
u32 len = vui->virtio_net_hdr_sz;
clib_memcpy_fast (&t->hdr, hdr,
len > hdr_desc->len ? hdr_desc->len : len);
}
}
static_always_inline u32
vhost_user_rx_discard_packet_packed (vlib_main_t * vm,
vhost_user_intf_t * vui,
vhost_user_vring_t * txvq,
u32 discard_max)
{
u32 discarded_packets = 0;
u16 mask = txvq->qsz_mask;
u16 desc_current, desc_head;
desc_head = desc_current = txvq->last_used_idx & mask;
/*
* On the RX side, each packet corresponds to one descriptor
* (it is the same whether it is a shallow descriptor, chained, or indirect).
* Therefore, discarding a packet is like discarding a descriptor.
*/
while ((discarded_packets != discard_max) &&
vhost_user_packed_desc_available (txvq, desc_current))
{
vhost_user_advance_last_avail_idx (txvq);
discarded_packets++;
desc_current = (desc_current + 1) & mask;
}
if (PREDICT_TRUE (discarded_packets))
vhost_user_mark_desc_consumed (vui, txvq, desc_head, discarded_packets);
return (discarded_packets);
}
static_always_inline u32
vhost_user_input_copy_packed (vhost_user_intf_t * vui, vhost_copy_t * cpy,
u16 copy_len, u32 * map_hint)
{
void *src0, *src1, *src2, *src3, *src4, *src5, *src6, *src7;
u8 bad;
u32 rc = VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR;
if (PREDICT_TRUE (copy_len >= 8))
{
src4 = map_guest_mem (vui, cpy[0].src, map_hint);
src5 = map_guest_mem (vui, cpy[1].src, map_hint);
src6 = map_guest_mem (vui, cpy[2].src, map_hint);
src7 = map_guest_mem (vui, cpy[3].src, map_hint);
bad = (src4 == 0) + (src5 == 0) + (src6 == 0) + (src7 == 0);
if (PREDICT_FALSE (bad))
goto one_by_one;
CLIB_PREFETCH (src4, 64, LOAD);
CLIB_PREFETCH (src5, 64, LOAD);
CLIB_PREFETCH (src6, 64, LOAD);
CLIB_PREFETCH (src7, 64, LOAD);
while (PREDICT_TRUE (copy_len >= 8))
{
src0 = src4;
src1 = src5;
src2 = src6;
src3 = src7;
src4 = map_guest_mem (vui, cpy[4].src, map_hint);
src5 = map_guest_mem (vui, cpy[5].src, map_hint);
src6 = map_guest_mem (vui, cpy[6].src, map_hint);
src7 = map_guest_mem (vui, cpy[7].src, map_hint);
bad = (src4 == 0) + (src5 == 0) + (src6 == 0) + (src7 == 0);
if (PREDICT_FALSE (bad))
break;
CLIB_PREFETCH (src4, 64, LOAD);
CLIB_PREFETCH (src5, 64, LOAD);
CLIB_PREFETCH (src6, 64, LOAD);
CLIB_PREFETCH (src7, 64, LOAD);
clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len);
clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len);
clib_memcpy_fast ((void *) cpy[2].dst, src2, cpy[2].len);
clib_memcpy_fast ((void *) cpy[3].dst, src3, cpy[3].len);
copy_len -= 4;
cpy += 4;
}
}
one_by_one:
while (copy_len)
{
if (PREDICT_FALSE (!(src0 = map_guest_mem (vui, cpy->src, map_hint))))
{
rc = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
break;
}
clib_memcpy_fast ((void *) cpy->dst, src0, cpy->len);
copy_len -= 1;
cpy += 1;
}
return rc;
}
static_always_inline u32
vhost_user_do_offload (vhost_user_intf_t * vui,
vring_packed_desc_t * desc_table, u16 desc_current,
u16 mask, vlib_buffer_t * b_head, u32 * map_hint)
{
u32 rc = VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR;
virtio_net_hdr_mrg_rxbuf_t *hdr;
u8 *b_data;
u32 desc_data_offset = vui->virtio_net_hdr_sz;
hdr = map_guest_mem (vui, desc_table[desc_current].addr, map_hint);
if (PREDICT_FALSE (hdr == 0))
rc = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
{
if (desc_data_offset == desc_table[desc_current].len)
{
desc_current = (desc_current + 1) & mask;
b_data =
map_guest_mem (vui, desc_table[desc_current].addr, map_hint);
if (PREDICT_FALSE (b_data == 0))
rc = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
else
vhost_user_handle_rx_offload (b_head, b_data, &hdr->hdr);
}
else
{
b_data = (u8 *) hdr + desc_data_offset;
vhost_user_handle_rx_offload (b_head, b_data, &hdr->hdr);
}
}
return rc;
}
static_always_inline u32
vhost_user_compute_buffers_required (u32 desc_len, u32 buffer_data_size)
{
div_t result;
u32 buffers_required;
if (PREDICT_TRUE (buffer_data_size == 2048))
{
buffers_required = desc_len >> 11;
if ((desc_len & 2047) != 0)
buffers_required++;
return (buffers_required);
}
if (desc_len < buffer_data_size)
return 1;
result = div (desc_len, buffer_data_size);
if (result.rem)
buffers_required = result.quot + 1;
else
buffers_required = result.quot;
return (buffers_required);
}
static_always_inline u32
vhost_user_compute_indirect_desc_len (vhost_user_intf_t * vui,
vhost_user_vring_t * txvq,
u32 buffer_data_size, u16 desc_current,
u32 * map_hint)
{
vring_packed_desc_t *desc_table = txvq->packed_desc;
u32 desc_len = 0;
u16 desc_data_offset = vui->virtio_net_hdr_sz;
u16 desc_idx = desc_current;
u32 n_descs;
n_descs = desc_table[desc_idx].len >> 4;
desc_table = map_guest_mem (vui, desc_table[desc_idx].addr, map_hint);
if (PREDICT_FALSE (desc_table == 0))
return 0;
for (desc_idx = 0; desc_idx < n_descs; desc_idx++)
desc_len += desc_table[desc_idx].len;
if (PREDICT_TRUE (desc_len > desc_data_offset))
desc_len -= desc_data_offset;
return vhost_user_compute_buffers_required (desc_len, buffer_data_size);
}
static_always_inline u32
vhost_user_compute_chained_desc_len (vhost_user_intf_t * vui,
vhost_user_vring_t * txvq,
u32 buffer_data_size, u16 * current,
u16 * n_left)
{
vring_packed_desc_t *desc_table = txvq->packed_desc;
u32 desc_len = 0;
u16 mask = txvq->qsz_mask;
while (desc_table[*current].flags & VIRTQ_DESC_F_NEXT)
{
desc_len += desc_table[*current].len;
(*n_left)++;
*current = (*current + 1) & mask;
vhost_user_advance_last_avail_idx (txvq);
}
desc_len += desc_table[*current].len;
(*n_left)++;
*current = (*current + 1) & mask;
vhost_user_advance_last_avail_idx (txvq);
if (PREDICT_TRUE (desc_len > vui->virtio_net_hdr_sz))
desc_len -= vui->virtio_net_hdr_sz;
return vhost_user_compute_buffers_required (desc_len, buffer_data_size);
}
static_always_inline void
vhost_user_assemble_packet (vring_packed_desc_t * desc_table,
u16 * desc_idx, vlib_buffer_t * b_head,
vlib_buffer_t ** b_current, u32 ** next,
vlib_buffer_t *** b, u32 * bi_current,
vhost_cpu_t * cpu, u16 * copy_len,
u32 * buffers_used, u32 buffers_required,
u32 * desc_data_offset, u32 buffer_data_size,
u16 mask)
{
u32 desc_data_l;
while (*desc_data_offset < desc_table[*desc_idx].len)
{
/* Get more output if necessary. Or end of packet. */
if (PREDICT_FALSE ((*b_current)->current_length == buffer_data_size))
{
/* Get next output */
u32 bi_next = **next;
(*next)++;
(*b_current)->next_buffer = bi_next;
(*b_current)->flags |= VLIB_BUFFER_NEXT_PRESENT;
*bi_current = bi_next;
*b_current = **b;
(*b)++;
(*buffers_used)++;
ASSERT (*buffers_used <= buffers_required);
}
/* Prepare a copy order executed later for the data */
ASSERT (*copy_len < VHOST_USER_COPY_ARRAY_N);
vhost_copy_t *cpy = &cpu->copy[*copy_len];
(*copy_len)++;
desc_data_l = desc_table[*desc_idx].len - *desc_data_offset;
cpy->len = buffer_data_size - (*b_current)->current_length;
cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len;
cpy->dst = (uword) (vlib_buffer_get_current (*b_current) +
(*b_current)->current_length);
cpy->src = desc_table[*desc_idx].addr + *desc_data_offset;
*desc_data_offset += cpy->len;
(*b_current)->current_length += cpy->len;
b_head->total_length_not_including_first_buffer += cpy->len;
}
*desc_idx = (*desc_idx + 1) & mask;;
*desc_data_offset = 0;
}
static_always_inline u32
vhost_user_if_input_packed (vlib_main_t * vm, vhost_user_main_t * vum,
vhost_user_intf_t * vui, u16 qid,
vlib_node_runtime_t * node,
vnet_hw_interface_rx_mode mode, u8 enable_csum)
{
vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
vnet_feature_main_t *fm = &feature_main;
u8 feature_arc_idx = fm->device_input_feature_arc_index;
u16 n_rx_packets = 0;
u32 n_rx_bytes = 0;
u16 n_left = 0;
u32 buffers_required = 0;
u32 n_left_to_next, *to_next;
u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
u32 n_trace = vlib_get_trace_count (vm, node);
u32 buffer_data_size = vlib_buffer_get_default_data_size (vm);
u32 map_hint = 0;
vhost_cpu_t *cpu = &vum->cpus[vm->thread_index];
u16 copy_len = 0;
u32 current_config_index = ~0;
u16 mask = txvq->qsz_mask;
u16 desc_current, desc_head, last_used_idx;
vring_packed_desc_t *desc_table = 0;
u32 n_descs_processed = 0;
u32 rv;
vlib_buffer_t **b;
u32 *next;
u32 buffers_used = 0;
u16 current, n_descs_to_process;
/* The descriptor table is not ready yet */
if (PREDICT_FALSE (txvq->packed_desc == 0))
goto done;
/* do we have pending interrupts ? */
vhost_user_vring_t *rxvq = &vui->vrings[VHOST_VRING_IDX_RX (qid)];
vhost_user_input_do_interrupt (vm, txvq, rxvq);
/*
* For adaptive mode, it is optimized to reduce interrupts.
* If the scheduler switches the input node to polling due
* to burst of traffic, we tell the driver no interrupt.
* When the traffic subsides, the scheduler switches the node back to
* interrupt mode. We must tell the driver we want interrupt.
*/
if (PREDICT_FALSE (mode == VNET_HW_INTERFACE_RX_MODE_ADAPTIVE))
{
if ((node->flags &
VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE) ||
!(node->flags &
VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE))
/* Tell driver we want notification */
txvq->used_event->flags = 0;
else
/* Tell driver we don't want notification */
txvq->used_event->flags = VRING_EVENT_F_DISABLE;
}
last_used_idx = txvq->last_used_idx & mask;
desc_head = desc_current = last_used_idx;
if (vhost_user_packed_desc_available (txvq, desc_current) == 0)
goto done;
if (PREDICT_FALSE (!vui->admin_up || !vui->is_ready || !(txvq->enabled)))
{
/*
* Discard input packet if interface is admin down or vring is not
* enabled.
* "For example, for a networking device, in the disabled state
* client must not supply any new RX packets, but must process
* and discard any TX packets."
*/
rv = vhost_user_rx_discard_packet_packed (vm, vui, txvq,
VHOST_USER_DOWN_DISCARD_COUNT);
vlib_error_count (vm, vhost_user_input_node.index,
VHOST_USER_INPUT_FUNC_ERROR_NOT_READY, rv);
goto done;
}
vhost_user_input_setup_frame (vm, node, vui, ¤t_config_index,
&next_index, &to_next, &n_left_to_next);
/*
* Compute n_left and total buffers needed
*/
desc_table = txvq->packed_desc;
current = desc_current;
while (vhost_user_packed_desc_available (txvq, current) &&
(n_left < VLIB_FRAME_SIZE))
{
if (desc_table[current].flags & VIRTQ_DESC_F_INDIRECT)
{
buffers_required +=
vhost_user_compute_indirect_desc_len (vui, txvq, buffer_data_size,
current, &map_hint);
n_left++;
current = (current + 1) & mask;
vhost_user_advance_last_avail_idx (txvq);
}
else
{
buffers_required +=
vhost_user_compute_chained_desc_len (vui, txvq, buffer_data_size,
¤t, &n_left);
}
}
/* Something is broken if we need more than 10000 buffers */
if (PREDICT_FALSE ((buffers_required == 0) || (buffers_required > 10000)))
{
rv = vhost_user_rx_discard_packet_packed (vm, vui, txvq, n_left);
vlib_error_count (vm, vhost_user_input_node.index,
VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, rv);
goto done;
}
vec_validate (cpu->to_next_list, buffers_required);
rv = vlib_buffer_alloc (vm, cpu->to_next_list, buffers_required);
if (PREDICT_FALSE (rv != buffers_required))
{
vlib_buffer_free (vm, cpu->to_next_list, rv);
rv = vhost_user_rx_discard_packet_packed (vm, vui, txvq, n_left);
vlib_error_count (vm, vhost_user_input_node.index,
VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, rv);
goto done;
}
next = cpu->to_next_list;
vec_validate (cpu->rx_buffers_pdesc, buffers_required);
vlib_get_buffers (vm, next, cpu->rx_buffers_pdesc, buffers_required);
b = cpu->rx_buffers_pdesc;
n_descs_processed = n_left;
while (n_left)
{
vlib_buffer_t *b_head, *b_current;
u32 bi_current;
u32 desc_data_offset;
u16 desc_idx = desc_current;
u32 n_descs;
desc_table = txvq->packed_desc;
to_next[0] = bi_current = next[0];
b_head = b_current = b[0];
b++;
buffers_used++;
ASSERT (buffers_used <= buffers_required);
to_next++;
next++;
n_left_to_next--;
/* The buffer should already be initialized */
b_head->total_length_not_including_first_buffer = 0;
b_head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
desc_data_offset = vui->virtio_net_hdr_sz;
n_descs_to_process = 1;
if (desc_table[desc_idx].flags & VIRTQ_DESC_F_INDIRECT)
{
n_descs = desc_table[desc_idx].len >> 4;
desc_table = map_guest_mem (vui, desc_table[desc_idx].addr,
&map_hint);
desc_idx = 0;
if (PREDICT_FALSE (desc_table == 0) ||
(enable_csum &&
(PREDICT_FALSE
(vhost_user_do_offload
(vui, desc_table, desc_idx, mask, b_head,
&map_hint) != VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))))
{
vlib_error_count (vm, node->node_index,
VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
to_next--;
next--;
n_left_to_next++;
buffers_used--;
b--;
goto out;
}
while (n_descs)
{
vhost_user_assemble_packet (desc_table, &desc_idx, b_head,
&b_current, &next, &b, &bi_current,
cpu, ©_len, &buffers_used,
buffers_required, &desc_data_offset,
buffer_data_size, mask);
n_descs--;
}
}
else
{
if (enable_csum)
{
rv = vhost_user_do_offload (vui, desc_table, desc_idx, mask,
b_head, &map_hint);
if (PREDICT_FALSE (rv != VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))
{
vlib_error_count (vm, node->node_index, rv, 1);
to_next--;
next--;
n_left_to_next++;
buffers_used--;
b--;
goto out;
}
}
/*
* For chained descriptor, we process all chains in a single while
* loop. So count how many descriptors in the chain.
*/
n_descs_to_process = 1;
while (desc_table[desc_idx].flags & VIRTQ_DESC_F_NEXT)
{
vhost_user_assemble_packet (desc_table, &desc_idx, b_head,
&b_current, &next, &b, &bi_current,
cpu, ©_len, &buffers_used,
buffers_required, &desc_data_offset,
buffer_data_size, mask);
n_descs_to_process++;
}
vhost_user_assemble_packet (desc_table, &desc_idx, b_head,
&b_current, &next, &b, &bi_current,
cpu, ©_len, &buffers_used,
buffers_required, &desc_data_offset,
buffer_data_size, mask);
}
n_rx_bytes += b_head->total_length_not_including_first_buffer;
n_rx_packets++;
b_head->total_length_not_including_first_buffer -=
b_head->current_length;
VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b_head);
vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index;
vnet_buffer (b_head)->sw_if_index[VLIB_TX] = ~0;
b_head->error = 0;
if (current_config_index != ~0)
{
b_head->current_config_index = current_config_index;
vnet_buffer (b_head)->feature_arc_index = feature_arc_idx;
}
out:
ASSERT (n_left >= n_descs_to_process);
n_left -= n_descs_to_process;
/* advance to next descrptor */
desc_current = (desc_current + n_descs_to_process) & mask;
/*
* Although separating memory copies from virtio ring parsing
* is beneficial, we can offer to perform the copies from time
* to time in order to free some space in the ring.
*/
if (PREDICT_FALSE (copy_len >= VHOST_USER_RX_COPY_THRESHOLD))
{
rv = vhost_user_input_copy_packed (vui, cpu->copy, copy_len,
&map_hint);
if (PREDICT_FALSE (rv != VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))
vlib_error_count (vm, node->node_index, rv, 1);
copy_len = 0;
}
}
vlib_put_next_frame (vm, node, next_index, n_left_to_next);
/* Do the memory copies */
rv = vhost_user_input_copy_packed (vui, cpu->copy, copy_len, &map_hint);
if (PREDICT_FALSE (rv != VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))
vlib_error_count (vm, node->node_index, rv, 1);
/* Must do the tracing before giving buffers back to driver */
if (PREDICT_FALSE (n_trace))
{
u32 left = n_rx_packets;
b = cpu->rx_buffers_pdesc;
while (n_trace && left)
{
vhost_trace_t *t0;
vlib_trace_buffer (vm, node, next_index, b[0],
/* follow_chain */ 0);
t0 = vlib_add_trace (vm, node, b[0], sizeof (t0[0]));
b++;
vhost_user_rx_trace_packed (t0, vui, qid, txvq, last_used_idx);
last_used_idx = (last_used_idx + 1) & mask;
n_trace--;
left--;
vlib_set_trace_count (vm, node, n_trace);
}
}
/*
* Give buffers back to driver.
*/
vhost_user_mark_desc_consumed (vui, txvq, desc_head, n_descs_processed);
/* interrupt (call) handling */
if ((txvq->callfd_idx != ~0) &&
(txvq->avail_event->flags != VRING_EVENT_F_DISABLE))
{
txvq->n_since_last_int += n_rx_packets;
if (txvq->n_since_last_int > vum->coalesce_frames)
vhost_user_send_call (vm, txvq);
}
/* increase rx counters */
vlib_increment_combined_counter
(vnet_main.interface_main.combined_sw_if_counters
+ VNET_INTERFACE_COUNTER_RX, vm->thread_index, vui->sw_if_index,
n_rx_packets, n_rx_bytes);
vnet_device_increment_rx_packets (vm->thread_index, n_rx_packets);
if (PREDICT_FALSE (buffers_used < buffers_required))
vlib_buffer_free (vm, next, buffers_required - buffers_used);
done:
return n_rx_packets;
}
VLIB_NODE_FN (vhost_user_input_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
vhost_user_main_t *vum = &vhost_user_main;
uword n_rx_packets = 0;
vhost_user_intf_t *vui;
vnet_device_input_runtime_t *rt =
(vnet_device_input_runtime_t *) node->runtime_data;
vnet_device_and_queue_t *dq;
vec_foreach (dq, rt->devices_and_queues)
{
if ((node->state == VLIB_NODE_STATE_POLLING) ||
clib_atomic_swap_acq_n (&dq->interrupt_pending, 0))
{
vui =
pool_elt_at_index (vum->vhost_user_interfaces, dq->dev_instance);
if (vhost_user_is_packed_ring_supported (vui))
{
if (vui->features & (1ULL << FEAT_VIRTIO_NET_F_CSUM))
n_rx_packets += vhost_user_if_input_packed (vm, vum, vui,
dq->queue_id, node,
dq->mode, 1);
else
n_rx_packets += vhost_user_if_input_packed (vm, vum, vui,
dq->queue_id, node,
dq->mode, 0);
}
else
{
if (vui->features & (1ULL << FEAT_VIRTIO_NET_F_CSUM))
n_rx_packets += vhost_user_if_input (vm, vum, vui, dq->queue_id,
node, dq->mode, 1);
else
n_rx_packets += vhost_user_if_input (vm, vum, vui, dq->queue_id,
node, dq->mode, 0);
}
}
}
return n_rx_packets;
}
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (vhost_user_input_node) = {
.type = VLIB_NODE_TYPE_INPUT,
.name = "vhost-user-input",
.sibling_of = "device-input",
.flags = VLIB_NODE_FLAG_TRACE_SUPPORTED,
/* Will be enabled if/when hardware is detected. */
.state = VLIB_NODE_STATE_DISABLED,
.format_buffer = format_ethernet_header_with_length,
.format_trace = format_vhost_trace,
.n_errors = VHOST_USER_INPUT_FUNC_N_ERROR,
.error_strings = vhost_user_input_func_error_strings,
};
/* *INDENT-ON* */
/*
* fd.io coding-style-patch-verification: ON
*
* Local Variables:
* eval: (c-set-style "gnu")
* End:
*/
