summaryrefslogtreecommitdiffstats
path: root/build
AgeCommit message (Expand)AuthorFilesLines
2021-08-31dpdk: bump to 21.08Fan Zhang1-1/+2
2021-08-16dpdk: patch i40e PMD to disable source pruningMatthew Smith1-0/+43
2021-07-28dpdk: fix cryptodev raw data path dequeueFan Zhang1-0/+40
2021-06-22build: vpp-ext-deps rpm version check may fail if system language is not Englishjiangxiaoming1-1/+1
2021-05-27dpdk: bump to 21.05Fan Zhang1-3/+3
2021-05-12quic: remove cmake --targetNathan Skrzypczak1-2/+1
2021-05-10rdma: bump to rdma-core v35.0Benoît Ganne1-17/+15
2021-05-07build: bump ipsecmb version to 1.0Damjan Marion1-4/+2
2021-04-22quic: update quicly to v0.1.3Mathias Raoul5-5/+81
2021-04-01dpdk: Update version to 21.02Kepka, Krzysztof1-4/+4
2021-01-22quic: quicly v0.1.2 updateMathias Raoul2-3/+63
2021-01-21dpdk: dpdk i40e patch cherry-pickChenmin Sun1-0/+134
2021-01-20crypto: improve ipsecmb build timePiotrX Kleski1-1/+1
2021-01-20rdma: fix pkg_config fileMohammed Hawari1-0/+3
2020-12-11dpdk: bump to dpdk 20.11Fan Zhang1-2/+2
2020-12-07dpdk: support mlx drivers linked with rdma-coreMohammed Hawari3-7/+22
2020-11-26crypto: improve ipsecmb engine performanceFan Zhang1-0/+3
2020-11-25dpdk: patch broken virtio macroMatthew Smith1-0/+42
2020-11-23crypto-ipsecmb: bump to intel-ipsec-mb version 0.55Dariusz Kazimierski2-3/+18
2020-11-19build: bump libbpf to v0.2Benoît Ganne1-1/+2
2020-11-19build: bump rdma-core to 31.1Benoît Ganne1-2/+2
2020-11-04build: better detection of libbpf dependenciesBenoît Ganne1-4/+6
2020-10-26dpdk: allow offline build by caching build-depsMohammed Hawari1-4/+6
2020-10-23build: fix typo in dpdk-20.08 patch #0003Dave Wallace1-2/+2
2020-10-17build: backport dpdk usertools support python3 only patchDave Wallace2-1/+213
2020-10-16misc: bump debian compatibility level to 10Damjan Marion1-1/+1
2020-10-14crypto: fixed ipsec_mb lib dependenciesPiotrX Kleski4-4/+24
2020-09-29dpdk: fix compileFan Zhang1-1/+3
2020-09-18dpdk: prevent linking dpdk against libbsdMohammed Hawari1-1/+16
2020-09-18dpdk: call the meson-based build instead of MakefilesMohammed Hawari2-271/+220
2020-09-09crypto: change cryptodev with new cryptodev APIFan Zhang1-0/+1606
2020-08-31af_xdp: AF_XDP input pluginBenoît Ganne2-1/+57
2020-08-26dpdk: bump to DPDK 20.08Damjan Marion1-2/+3
2020-08-25rdma: bump rdma-core version to v31.0Benoît Ganne1-3/+2
2020-05-27dpdk: bump to DPDK v20.05Damjan Marion1-2/+6
2020-05-13dpdk: DPDK 20.05 iavf fdir bug-fix patch cherry pickChenmin Sun1-0/+70
2020-05-08dpdk: fix compiling issue with clangLijian.Zhang1-1/+2
2020-04-22dpdk: DPDK 20.05 iavf flow director backporting to DPDK 20.02Chenmin Sun17-0/+6436
2020-03-27quic: Check quicly version tag at compile timeMathiasRaoul13-534/+23
2020-03-22dpdk: false link down issue with ixgbe NICLijian.Zhang1-0/+66
2020-03-12rdma: cleanup rdma-core buildBenoît Ganne1-12/+9
2020-02-26rdma: bunp rdma-core version to v28.0Damjan Marion1-1/+2
2020-02-26dpdk: bump dpdk version to 20.02Damjan Marion1-2/+2
2020-02-04dpdk: patch ixgbe driver to solve race conditionMatthew Smith1-0/+49
2020-01-31quic: update quicly to v0.0.10-vppMathiasRaoul2-2/+62
2020-01-06quic: update quicly to v0.0.9-vppMathiasRaoul2-2/+62
2019-12-17quic: update quicly to v0.0.8-vppMathiasRaoul2-2/+62
2019-12-06quic: update quicly to v0.0.7-vppMathiasRaoul2-2/+62
2019-11-25quic: update quicly to v0.0.6-vppMathiasRaoul3-2/+95
2019-11-22crypto-ipsecmb: bump to intel-ipsec-mb version 0.53Fan Zhang1-2/+3
>922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 
/*
 * Copyright (c) 2015 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/*
 * buffer_funcs.h: VLIB buffer related functions/inlines
 *
 * Copyright (c) 2008 Eliot Dresselhaus
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#ifndef included_vlib_buffer_funcs_h
#define included_vlib_buffer_funcs_h

#include <vppinfra/hash.h>
#include <vppinfra/fifo.h>
#include <vlib/buffer.h>
#include <vlib/physmem_funcs.h>
#include <vlib/main.h>
#include <vlib/node.h>

/** \file
    vlib buffer access methods.
*/

always_inline void
vlib_buffer_validate (vlib_main_t * vm, vlib_buffer_t * b)
{
  vlib_buffer_main_t *bm = vm->buffer_main;
  vlib_buffer_pool_t *bp;

  /* reference count in allocated buffer always must be 1 or higher */
  ASSERT (b->ref_count > 0);

  /* verify that buffer pool index is valid */
  bp = vec_elt_at_index (bm->buffer_pools, b->buffer_pool_index);
  ASSERT (pointer_to_uword (b) >= bp->start);
  ASSERT (pointer_to_uword (b) < bp->start + bp->size -
	  (bp->data_size + sizeof (vlib_buffer_t)));
}

always_inline void *
vlib_buffer_ptr_from_index (uword buffer_mem_start, u32 buffer_index,
			    uword offset)
{
  offset += ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
  return uword_to_pointer (buffer_mem_start + offset, vlib_buffer_t *);
}

/** \brief Translate buffer index into buffer pointer

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffer_index - (u32) buffer index
    @return - (vlib_buffer_t *) buffer pointer
*/
always_inline vlib_buffer_t *
vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
{
  vlib_buffer_main_t *bm = vm->buffer_main;
  vlib_buffer_t *b;

  b = vlib_buffer_ptr_from_index (bm->buffer_mem_start, buffer_index, 0);
  vlib_buffer_validate (vm, b);
  return b;
}

static_always_inline u32
vlib_buffer_get_default_data_size (vlib_main_t * vm)
{
  return vm->buffer_main->default_data_size;
}

static_always_inline void
vlib_buffer_copy_indices (u32 * dst, u32 * src, u32 n_indices)
{
#if defined(CLIB_HAVE_VEC512)
  while (n_indices >= 16)
    {
      u32x16_store_unaligned (u32x16_load_unaligned (src), dst);
      dst += 16;
      src += 16;
      n_indices -= 16;
    }
#endif

#if defined(CLIB_HAVE_VEC256)
  while (n_indices >= 8)
    {
      u32x8_store_unaligned (u32x8_load_unaligned (src), dst);
      dst += 8;
      src += 8;
      n_indices -= 8;
    }
#endif

#if defined(CLIB_HAVE_VEC128)
  while (n_indices >= 4)
    {
      u32x4_store_unaligned (u32x4_load_unaligned (src), dst);
      dst += 4;
      src += 4;
      n_indices -= 4;
    }
#endif

  while (n_indices)
    {
      dst[0] = src[0];
      dst += 1;
      src += 1;
      n_indices -= 1;
    }
}

STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64);
static_always_inline void
vlib_buffer_copy_template (vlib_buffer_t * b, vlib_buffer_t * bt)
{
#if defined CLIB_HAVE_VEC512
  b->as_u8x64[0] = bt->as_u8x64[0];
#elif defined (CLIB_HAVE_VEC256)
  b->as_u8x32[0] = bt->as_u8x32[0];
  b->as_u8x32[1] = bt->as_u8x32[1];
#elif defined (CLIB_HAVE_VEC128)
  b->as_u8x16[0] = bt->as_u8x16[0];
  b->as_u8x16[1] = bt->as_u8x16[1];
  b->as_u8x16[2] = bt->as_u8x16[2];
  b->as_u8x16[3] = bt->as_u8x16[3];
#else
  clib_memcpy_fast (b, bt, 64);
#endif
}

always_inline u8
vlib_buffer_pool_get_default_for_numa (vlib_main_t * vm, u32 numa_node)
{
  ASSERT (numa_node < VLIB_BUFFER_MAX_NUMA_NODES);
  return vm->buffer_main->default_buffer_pool_index_for_numa[numa_node];
}

/** \brief Translate array of buffer indices into buffer pointers with offset

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param bi - (u32 *) array of buffer indices
    @param b - (void **) array to store buffer pointers
    @param count - (uword) number of elements
    @param offset - (i32) offset applied to each pointer
*/
static_always_inline void
vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count,
			      i32 offset)
{
  uword buffer_mem_start = vm->buffer_main->buffer_mem_start;
#ifdef CLIB_HAVE_VEC256
  u64x4 off = u64x4_splat (buffer_mem_start + offset);
  /* if count is not const, compiler will not unroll while loop
     se we maintain two-in-parallel variant */
  while (count >= 8)
    {
      u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
      u64x4 b1 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi + 4));
      /* shift and add to get vlib_buffer_t pointer */
      u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
      u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4);
      b += 8;
      bi += 8;
      count -= 8;
    }
#endif
  while (count >= 4)
    {
#ifdef CLIB_HAVE_VEC256
      u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
      /* shift and add to get vlib_buffer_t pointer */
      u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
#elif defined (CLIB_HAVE_VEC128)
      u64x2 off = u64x2_splat (buffer_mem_start + offset);
      u32x4 bi4 = u32x4_load_unaligned (bi);
      u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4);
#if defined (__aarch64__)
      u64x2 b1 = u32x4_extend_to_u64x2_high ((u32x4) bi4);
#else
      bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1);
      u64x2 b1 = u32x4_extend_to_u64x2 ((u32x4) bi4);
#endif
      u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
      u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2);
#else
      b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
      b[1] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[1], offset);
      b[2] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[2], offset);
      b[3] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[3], offset);
#endif
      b += 4;
      bi += 4;
      count -= 4;
    }
  while (count)
    {
      b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
      b += 1;
      bi += 1;
      count -= 1;
    }
}

/** \brief Translate array of buffer indices into buffer pointers

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param bi - (u32 *) array of buffer indices
    @param b - (vlib_buffer_t **) array to store buffer pointers
    @param count - (uword) number of elements
*/

static_always_inline void
vlib_get_buffers (vlib_main_t * vm, u32 * bi, vlib_buffer_t ** b, int count)
{
  vlib_get_buffers_with_offset (vm, bi, (void **) b, count, 0);
}

/** \brief Translate buffer pointer into buffer index

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param p - (void *) buffer pointer
    @return - (u32) buffer index
*/

always_inline u32
vlib_get_buffer_index (vlib_main_t * vm, void *p)
{
  vlib_buffer_main_t *bm = vm->buffer_main;
  uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
  ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
  ASSERT (offset < bm->buffer_mem_size);
  ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
  return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
}

/** \brief Translate array of buffer pointers into buffer indices with offset

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param b - (void **) array of buffer pointers
    @param bi - (u32 *) array to store buffer indices
    @param count - (uword) number of elements
    @param offset - (i32) offset applied to each pointer
*/
static_always_inline void
vlib_get_buffer_indices_with_offset (vlib_main_t * vm, void **b, u32 * bi,
				     uword count, i32 offset)
{
#ifdef CLIB_HAVE_VEC256
  u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
  u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset);

  while (count >= 8)
    {
      /* load 4 pointers into 256-bit register */
      u64x4 v0 = u64x4_load_unaligned (b);
      u64x4 v1 = u64x4_load_unaligned (b + 4);
      u32x8 v2, v3;

      v0 -= off4;
      v1 -= off4;

      v0 >>= CLIB_LOG2_CACHE_LINE_BYTES;
      v1 >>= CLIB_LOG2_CACHE_LINE_BYTES;

      /* permute 256-bit register so lower u32s of each buffer index are
       * placed into lower 128-bits */
      v2 = u32x8_permute ((u32x8) v0, mask);
      v3 = u32x8_permute ((u32x8) v1, mask);

      /* extract lower 128-bits and save them to the array of buffer indices */
      u32x4_store_unaligned (u32x8_extract_lo (v2), bi);
      u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4);
      bi += 8;
      b += 8;
      count -= 8;
    }
#endif
  while (count >= 4)
    {
      /* equivalent non-nector implementation */
      bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
      bi[1] = vlib_get_buffer_index (vm, ((u8 *) b[1]) + offset);
      bi[2] = vlib_get_buffer_index (vm, ((u8 *) b[2]) + offset);
      bi[3] = vlib_get_buffer_index (vm, ((u8 *) b[3]) + offset);
      bi += 4;
      b += 4;
      count -= 4;
    }
  while (count)
    {
      bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
      bi += 1;
      b += 1;
      count -= 1;
    }
}

/** \brief Translate array of buffer pointers into buffer indices

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param b - (vlib_buffer_t **) array of buffer pointers
    @param bi - (u32 *) array to store buffer indices
    @param count - (uword) number of elements
*/
static_always_inline void
vlib_get_buffer_indices (vlib_main_t * vm, vlib_buffer_t ** b, u32 * bi,
			 uword count)
{
  vlib_get_buffer_indices_with_offset (vm, (void **) b, bi, count, 0);
}

/** \brief Get next buffer in buffer linklist, or zero for end of list.

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param b - (void *) buffer pointer
    @return - (vlib_buffer_t *) next buffer, or NULL
*/
always_inline vlib_buffer_t *
vlib_get_next_buffer (vlib_main_t * vm, vlib_buffer_t * b)
{
  return (b->flags & VLIB_BUFFER_NEXT_PRESENT
	  ? vlib_get_buffer (vm, b->next_buffer) : 0);
}

uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
					     vlib_buffer_t * b_first);

/** \brief Get length in bytes of the buffer chain

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param b - (void *) buffer pointer
    @return - (uword) length of buffer chain
*/
always_inline uword
vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b)
{
  uword len = b->current_length;

  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
    return len;

  if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
    return len + b->total_length_not_including_first_buffer;

  return vlib_buffer_length_in_chain_slow_path (vm, b);
}

/** \brief Get length in bytes of the buffer index buffer chain

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param bi - (u32) buffer index
    @return - (uword) length of buffer chain
*/
always_inline uword
vlib_buffer_index_length_in_chain (vlib_main_t * vm, u32 bi)
{
  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
  return vlib_buffer_length_in_chain (vm, b);
}

/** \brief Copy buffer contents to memory

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffer_index - (u32) buffer index
    @param contents - (u8 *) memory, <strong>must be large enough</strong>
    @return - (uword) length of buffer chain
*/
always_inline uword
vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
{
  uword content_len = 0;
  uword l;
  vlib_buffer_t *b;

  while (1)
    {
      b = vlib_get_buffer (vm, buffer_index);
      l = b->current_length;
      clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
      content_len += l;
      if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
	break;
      buffer_index = b->next_buffer;
    }

  return content_len;
}

always_inline uword
vlib_buffer_get_pa (vlib_main_t * vm, vlib_buffer_t * b)
{
  return vlib_physmem_get_pa (vm, b->data);
}

always_inline uword
vlib_buffer_get_current_pa (vlib_main_t * vm, vlib_buffer_t * b)
{
  return vlib_buffer_get_pa (vm, b) + b->current_data;
}

/** \brief Prefetch buffer metadata by buffer index
    The first 64 bytes of buffer contains most header information

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param bi - (u32) buffer index
    @param type - LOAD, STORE. In most cases, STORE is the right answer
*/
/* Prefetch buffer header given index. */
#define vlib_prefetch_buffer_with_index(vm,bi,type)	\
  do {							\
    vlib_buffer_t * _b = vlib_get_buffer (vm, bi);	\
    vlib_prefetch_buffer_header (_b, type);		\
  } while (0)

typedef enum
{
  /* Index is unknown. */
  VLIB_BUFFER_UNKNOWN,

  /* Index is known and free/allocated. */
  VLIB_BUFFER_KNOWN_FREE,
  VLIB_BUFFER_KNOWN_ALLOCATED,
} vlib_buffer_known_state_t;

void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
				      uword n_buffers,
				      vlib_buffer_known_state_t
				      expected_state);

always_inline vlib_buffer_known_state_t
vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
{
  vlib_buffer_main_t *bm = vm->buffer_main;

  clib_spinlock_lock (&bm->buffer_known_hash_lockp);
  uword *p = hash_get (bm->buffer_known_hash, buffer_index);
  clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
  return p ? p[0] : VLIB_BUFFER_UNKNOWN;
}

/* Validates sanity of a single buffer.
   Returns format'ed vector with error message if any. */
u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
			  uword follow_chain);

static_always_inline vlib_buffer_pool_t *
vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index)
{
  vlib_buffer_main_t *bm = vm->buffer_main;
  return vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
}

static_always_inline uword
vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers,
		      u32 n_buffers)
{
  vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
  u32 len;

  ASSERT (bp->buffers);

  clib_spinlock_lock (&bp->lock);
  len = vec_len (bp->buffers);
  if (PREDICT_TRUE (n_buffers < len))
    {
      len -= n_buffers;
      vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers);
      _vec_len (bp->buffers) = len;
      clib_spinlock_unlock (&bp->lock);
      return n_buffers;
    }
  else
    {
      vlib_buffer_copy_indices (buffers, bp->buffers, len);
      _vec_len (bp->buffers) = 0;
      clib_spinlock_unlock (&bp->lock);
      return len;
    }
}


/** \brief Allocate buffers from specific pool into supplied array

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffers - (u32 * ) buffer index array
    @param n_buffers - (u32) number of buffers requested
    @return - (u32) number of buffers actually allocated, may be
    less than the number requested or zero
*/

always_inline u32
vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
			     u8 buffer_pool_index)
{
  vlib_buffer_main_t *bm = vm->buffer_main;
  vlib_buffer_pool_t *bp;
  vlib_buffer_pool_thread_t *bpt;
  u32 *src, *dst, len, n_left;

  bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
  bpt = vec_elt_at_index (bp->threads, vm->thread_index);

  dst = buffers;
  n_left = n_buffers;
  len = vec_len (bpt->cached_buffers);

  /* per-thread cache contains enough buffers */
  if (len >= n_buffers)
    {
      src = bpt->cached_buffers + len - n_buffers;
      vlib_buffer_copy_indices (dst, src, n_buffers);
      _vec_len (bpt->cached_buffers) -= n_buffers;

      if (CLIB_DEBUG > 0)
	vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
					 VLIB_BUFFER_KNOWN_FREE);
      return n_buffers;
    }

  /* take everything available in the cache */
  if (len)
    {
      vlib_buffer_copy_indices (dst, bpt->cached_buffers, len);
      _vec_len (bpt->cached_buffers) = 0;
      dst += len;
      n_left -= len;
    }

  len = round_pow2 (n_left, 32);
  vec_validate_aligned (bpt->cached_buffers, len - 1, CLIB_CACHE_LINE_BYTES);
  len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers,
			      len);
  _vec_len (bpt->cached_buffers) = len;

  if (len)
    {
      u32 n_copy = clib_min (len, n_left);
      src = bpt->cached_buffers + len - n_copy;
      vlib_buffer_copy_indices (dst, src, n_copy);
      _vec_len (bpt->cached_buffers) -= n_copy;
      n_left -= n_copy;
    }

  n_buffers -= n_left;

  /* Verify that buffers are known free. */
  if (CLIB_DEBUG > 0)
    vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
				     VLIB_BUFFER_KNOWN_FREE);

  return n_buffers;
}

/** \brief Allocate buffers from specific numa node into supplied array

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffers - (u32 * ) buffer index array
    @param n_buffers - (u32) number of buffers requested
    @param numa_node - (u32) numa node
    @return - (u32) number of buffers actually allocated, may be
    less than the number requested or zero
*/
always_inline u32
vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
			   u32 numa_node)
{
  u8 index = vlib_buffer_pool_get_default_for_numa (vm, numa_node);
  return vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, index);
}

/** \brief Allocate buffers into supplied array

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffers - (u32 * ) buffer index array
    @param n_buffers - (u32) number of buffers requested
    @return - (u32) number of buffers actually allocated, may be
    less than the number requested or zero
*/

always_inline u32
vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
{
  return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node);
}

/** \brief Allocate buffers into ring

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffers - (u32 * ) buffer index ring
    @param start - (u32) first slot in the ring
    @param ring_size - (u32) ring size
    @param n_buffers - (u32) number of buffers requested
    @return - (u32) number of buffers actually allocated, may be
    less than the number requested or zero
*/
always_inline u32
vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start,
			   u32 ring_size, u32 n_buffers)
{
  u32 n_alloc;

  ASSERT (n_buffers <= ring_size);

  if (PREDICT_TRUE (start + n_buffers <= ring_size))
    return vlib_buffer_alloc (vm, ring + start, n_buffers);

  n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start);

  if (PREDICT_TRUE (n_alloc == ring_size - start))
    n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc);

  return n_alloc;
}

/** \brief Allocate buffers into ring from specific buffer pool

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffers - (u32 * ) buffer index ring
    @param start - (u32) first slot in the ring
    @param ring_size - (u32) ring size
    @param n_buffers - (u32) number of buffers requested
    @return - (u32) number of buffers actually allocated, may be
    less than the number requested or zero
*/
always_inline u32
vlib_buffer_alloc_to_ring_from_pool (vlib_main_t * vm, u32 * ring, u32 start,
				     u32 ring_size, u32 n_buffers,
				     u8 buffer_pool_index)
{
  u32 n_alloc;

  ASSERT (n_buffers <= ring_size);

  if (PREDICT_TRUE (start + n_buffers <= ring_size))
    return vlib_buffer_alloc_from_pool (vm, ring + start, n_buffers,
					buffer_pool_index);

  n_alloc = vlib_buffer_alloc_from_pool (vm, ring + start, ring_size - start,
					 buffer_pool_index);

  if (PREDICT_TRUE (n_alloc == ring_size - start))
    n_alloc += vlib_buffer_alloc_from_pool (vm, ring, n_buffers - n_alloc,
					    buffer_pool_index);

  return n_alloc;
}

static_always_inline void
vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index,
		      u32 * buffers, u32 n_buffers)
{
  vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
  vlib_buffer_pool_thread_t *bpt =
    vec_elt_at_index (bp->threads, vm->thread_index);

  if (CLIB_DEBUG > 0)
    vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
				     VLIB_BUFFER_KNOWN_ALLOCATED);

  vec_add_aligned (bpt->cached_buffers, buffers, n_buffers,
		   CLIB_CACHE_LINE_BYTES);

  if (vec_len (bpt->cached_buffers) > 4 * VLIB_FRAME_SIZE)
    {
      clib_spinlock_lock (&bp->lock);
      /* keep last stored buffers, as they are more likely hot in the cache */
      vec_add_aligned (bp->buffers, bpt->cached_buffers, VLIB_FRAME_SIZE,
		       CLIB_CACHE_LINE_BYTES);
      vec_delete (bpt->cached_buffers, VLIB_FRAME_SIZE, 0);
      bpt->n_alloc -= VLIB_FRAME_SIZE;
      clib_spinlock_unlock (&bp->lock);
    }
}

static_always_inline void
vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
			 int maybe_next)
{
  const int queue_size = 128;
  vlib_buffer_pool_t *bp = 0;
  u8 buffer_pool_index = ~0;
  u32 n_queue = 0, queue[queue_size + 4];
  vlib_buffer_t bt = { };
#if defined(CLIB_HAVE_VEC128)
  vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 };
  vlib_buffer_t bpi_vec = {.buffer_pool_index = ~0 };
  vlib_buffer_t flags_refs_mask = {
    .flags = VLIB_BUFFER_NEXT_PRESENT,
    .ref_count = ~1
  };
#endif

  while (n_buffers)
    {
      vlib_buffer_t *b[8];
      u32 bi, sum = 0, flags, next;

      if (n_buffers < 12)
	goto one_by_one;

      vlib_get_buffers (vm, buffers, b, 4);
      vlib_get_buffers (vm, buffers + 8, b + 4, 4);

      vlib_prefetch_buffer_header (b[4], LOAD);
      vlib_prefetch_buffer_header (b[5], LOAD);
      vlib_prefetch_buffer_header (b[6], LOAD);
      vlib_prefetch_buffer_header (b[7], LOAD);

#if defined(CLIB_HAVE_VEC128)
      u8x16 p0, p1, p2, p3, r;
      p0 = u8x16_load_unaligned (b[0]);
      p1 = u8x16_load_unaligned (b[1]);
      p2 = u8x16_load_unaligned (b[2]);
      p3 = u8x16_load_unaligned (b[3]);

      r = p0 ^ bpi_vec.as_u8x16[0];
      r |= p1 ^ bpi_vec.as_u8x16[0];
      r |= p2 ^ bpi_vec.as_u8x16[0];
      r |= p3 ^ bpi_vec.as_u8x16[0];
      r &= bpi_mask.as_u8x16[0];
      r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0];

      sum = !u8x16_is_all_zero (r);
#else
      sum |= b[0]->flags;
      sum |= b[1]->flags;
      sum |= b[2]->flags;
      sum |= b[3]->flags;
      sum &= VLIB_BUFFER_NEXT_PRESENT;
      sum += b[0]->ref_count - 1;
      sum += b[1]->ref_count - 1;
      sum += b[2]->ref_count - 1;
      sum += b[3]->ref_count - 1;
      sum |= b[0]->buffer_pool_index ^ buffer_pool_index;
      sum |= b[1]->buffer_pool_index ^ buffer_pool_index;
      sum |= b[2]->buffer_pool_index ^ buffer_pool_index;
      sum |= b[3]->buffer_pool_index ^ buffer_pool_index;
#endif

      if (sum)
	goto one_by_one;

      vlib_buffer_copy_indices (queue + n_queue, buffers, 4);
      vlib_buffer_copy_template (b[0], &bt);
      vlib_buffer_copy_template (b[1], &bt);
      vlib_buffer_copy_template (b[2], &bt);
      vlib_buffer_copy_template (b[3], &bt);
      n_queue += 4;

      vlib_buffer_validate (vm, b[0]);
      vlib_buffer_validate (vm, b[1]);
      vlib_buffer_validate (vm, b[2]);
      vlib_buffer_validate (vm, b[3]);

      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]);
      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]);
      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]);

      if (n_queue >= queue_size)
	{
	  vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
	  n_queue = 0;
	}
      buffers += 4;
      n_buffers -= 4;
      continue;

    one_by_one:
      bi = buffers[0];

    next_in_chain:
      b[0] = vlib_get_buffer (vm, bi);
      flags = b[0]->flags;
      next = b[0]->next_buffer;

      if (PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index))
	{

	  if (n_queue)
	    {
	      vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
	      n_queue = 0;
	    }

	  buffer_pool_index = b[0]->buffer_pool_index;
#if defined(CLIB_HAVE_VEC128)
	  bpi_vec.buffer_pool_index = buffer_pool_index;
#endif
	  bp = vlib_get_buffer_pool (vm, buffer_pool_index);
	  vlib_buffer_copy_template (&bt, &bp->buffer_template);
	}

      vlib_buffer_validate (vm, b[0]);

      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);

      if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0)
	{
	  vlib_buffer_copy_template (b[0], &bt);
	  queue[n_queue++] = bi;
	}

      if (n_queue == queue_size)
	{
	  vlib_buffer_pool_put (vm, buffer_pool_index, queue, queue_size);
	  n_queue = 0;
	}

      if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
	{
	  bi = next;
	  goto next_in_chain;
	}

      buffers++;
      n_buffers--;
    }

  if (n_queue)
    vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
}


/** \brief Free buffers
    Frees the entire buffer chain for each buffer

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffers - (u32 * ) buffer index array
    @param n_buffers - (u32) number of buffers to free

*/
always_inline void
vlib_buffer_free (vlib_main_t * vm,
		  /* pointer to first buffer */
		  u32 * buffers,
		  /* number of buffers to free */
		  u32 n_buffers)
{
  vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 1);
}

/** \brief Free buffers, does not free the buffer chain for each buffer

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffers - (u32 * ) buffer index array
    @param n_buffers - (u32) number of buffers to free

*/
always_inline void
vlib_buffer_free_no_next (vlib_main_t * vm,
			  /* pointer to first buffer */
			  u32 * buffers,
			  /* number of buffers to free */
			  u32 n_buffers)
{
  vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 0);
}

/** \brief Free one buffer
    Shorthand to free a single buffer chain.

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffer_index - (u32) buffer index to free
*/
always_inline void
vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index)
{
  vlib_buffer_free_inline (vm, &buffer_index, 1, /* maybe next */ 1);
}

/** \brief Free buffers from ring

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffers - (u32 * ) buffer index ring
    @param start - (u32) first slot in the ring
    @param ring_size - (u32) ring size
    @param n_buffers - (u32) number of buffers
*/
always_inline void
vlib_buffer_free_from_ring (vlib_main_t * vm, u32 * ring, u32 start,
			    u32 ring_size, u32 n_buffers)
{
  ASSERT (n_buffers <= ring_size);

  if (PREDICT_TRUE (start + n_buffers <= ring_size))
    {
      vlib_buffer_free (vm, ring + start, n_buffers);
    }
  else
    {
      vlib_buffer_free (vm, ring + start, ring_size - start);
      vlib_buffer_free (vm, ring, n_buffers - (ring_size - start));
    }
}

/** \brief Free buffers from ring without freeing tail buffers

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param buffers - (u32 * ) buffer index ring
    @param start - (u32) first slot in the ring
    @param ring_size - (u32) ring size
    @param n_buffers - (u32) number of buffers
*/
always_inline void
vlib_buffer_free_from_ring_no_next (vlib_main_t * vm, u32 * ring, u32 start,
				    u32 ring_size, u32 n_buffers)
{
  ASSERT (n_buffers <= ring_size);

  if (PREDICT_TRUE (start + n_buffers <= ring_size))
    {
      vlib_buffer_free_no_next (vm, ring + start, n_buffers);
    }
  else
    {
      vlib_buffer_free_no_next (vm, ring + start, ring_size - start);
      vlib_buffer_free_no_next (vm, ring, n_buffers - (ring_size - start));
    }
}

/* Append given data to end of buffer, possibly allocating new buffers. */
int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
			  u32 n_data_bytes);

/* Define vlib_buffer and vnet_buffer flags bits preserved for copy/clone */
#define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK                     	\
  (VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID |	\
   VLIB_BUFFER_IS_TRACED | ~VLIB_BUFFER_FLAGS_ALL)

/* duplicate all buffers in chain */
always_inline vlib_buffer_t *
vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
{
  vlib_buffer_t *s, *d, *fd;
  uword n_alloc, n_buffers = 1;
  u32 flag_mask = VLIB_BUFFER_COPY_CLONE_FLAGS_MASK;
  int i;

  s = b;
  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
    {
      n_buffers++;
      s = vlib_get_buffer (vm, s->next_buffer);
    }
  u32 new_buffers[n_buffers];

  n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);

  /* No guarantee that we'll get all the buffers we asked for */
  if (PREDICT_FALSE (n_alloc < n_buffers))
    {
      if (n_alloc > 0)
	vlib_buffer_free (vm, new_buffers, n_alloc);
      return 0;
    }

  /* 1st segment */
  s = b;
  fd = d = vlib_get_buffer (vm, new_buffers[0]);
  d->current_data = s->current_data;
  d->current_length = s->current_length;
  d->flags = s->flags & flag_mask;
  d->total_length_not_including_first_buffer =
    s->total_length_not_including_first_buffer;
  clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
  clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
  clib_memcpy_fast (vlib_buffer_get_current (d),
		    vlib_buffer_get_current (s), s->current_length);

  /* next segments */
  for (i = 1; i < n_buffers; i++)
    {
      /* previous */
      d->next_buffer = new_buffers[i];
      /* current */
      s = vlib_get_buffer (vm, s->next_buffer);
      d = vlib_get_buffer (vm, new_buffers[i]);
      d->current_data = s->current_data;
      d->current_length = s->current_length;
      clib_memcpy_fast (vlib_buffer_get_current (d),
			vlib_buffer_get_current (s), s->current_length);
      d->flags = s->flags & flag_mask;
    }

  return fd;
}

/* duplicate first buffer in chain */
always_inline vlib_buffer_t *
vlib_buffer_copy_no_chain (vlib_main_t * vm, vlib_buffer_t * b, u32 * di)
{
  vlib_buffer_t *d;

  if ((vlib_buffer_alloc (vm, di, 1)) != 1)
    return 0;

  d = vlib_get_buffer (vm, *di);
  /* 1st segment */
  d->current_data = b->current_data;
  d->current_length = b->current_length;
  clib_memcpy_fast (d->opaque, b->opaque, sizeof (b->opaque));
  clib_memcpy_fast (d->opaque2, b->opaque2, sizeof (b->opaque2));
  clib_memcpy_fast (vlib_buffer_get_current (d),
		    vlib_buffer_get_current (b), b->current_length);

  return d;
}

/*  \brief Move packet from current position to offset position in buffer.
    Only work for small packet using one buffer with room to fit the move
    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param b -  (vlib_buffer_t *) pointer to buffer
    @param offset - (i16) position to move the packet in buffer
 */
always_inline void
vlib_buffer_move (vlib_main_t * vm, vlib_buffer_t * b, i16 offset)
{
  ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
  ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
  ASSERT (offset + b->current_length <
	  vlib_buffer_get_default_data_size (vm));

  u8 *source = vlib_buffer_get_current (b);
  b->current_data = offset;
  u8 *destination = vlib_buffer_get_current (b);
  u16 length = b->current_length;

  if (source + length <= destination)	/* no overlap */
    clib_memcpy_fast (destination, source, length);
  else
    memmove (destination, source, length);
}

/** \brief Create a maximum of 256 clones of buffer and store them
    in the supplied array

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param src_buffer - (u32) source buffer index
    @param buffers - (u32 * ) buffer index array
    @param n_buffers - (u16) number of buffer clones requested (<=256)
    @param head_end_offset - (u16) offset relative to current position
           where packet head ends
    @param offset - (i16) copy packet head at current position if 0,
           else at offset position to change headroom space as specified
    @return - (u16) number of buffers actually cloned, may be
    less than the number requested or zero
*/
always_inline u16
vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
		       u16 n_buffers, u16 head_end_offset, i16 offset)
{
  u16 i;
  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);

  ASSERT (s->ref_count == 1);
  ASSERT (n_buffers);
  ASSERT (n_buffers <= 256);
  ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
  ASSERT ((offset + head_end_offset) <
	  vlib_buffer_get_default_data_size (vm));

  if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
    {
      buffers[0] = src_buffer;
      if (offset)
	vlib_buffer_move (vm, s, offset);

      for (i = 1; i < n_buffers; i++)
	{
	  vlib_buffer_t *d;
	  d = vlib_buffer_copy (vm, s);
	  if (d == 0)
	    return i;
	  buffers[i] = vlib_get_buffer_index (vm, d);

	}
      return n_buffers;
    }

  if (PREDICT_FALSE ((n_buffers == 1) && (offset == 0)))
    {
      buffers[0] = src_buffer;
      return 1;
    }

  n_buffers = vlib_buffer_alloc_from_pool (vm, buffers, n_buffers,
					   s->buffer_pool_index);

  for (i = 0; i < n_buffers; i++)
    {
      vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
      if (offset)
	d->current_data = offset;
      else
	d->current_data = s->current_data;

      d->current_length = head_end_offset;
      ASSERT (d->buffer_pool_index == s->buffer_pool_index);

      d->total_length_not_including_first_buffer = s->current_length -
	head_end_offset;
      if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT))
	{
	  d->total_length_not_including_first_buffer +=
	    s->total_length_not_including_first_buffer;
	}
      d->flags = (s->flags & VLIB_BUFFER_COPY_CLONE_FLAGS_MASK) |
	VLIB_BUFFER_NEXT_PRESENT;
      clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
      clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
      clib_memcpy_fast (vlib_buffer_get_current (d),
			vlib_buffer_get_current (s), head_end_offset);
      d->next_buffer = src_buffer;
    }
  vlib_buffer_advance (s, head_end_offset);
  s->ref_count = n_buffers;
  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
    {
      s = vlib_get_buffer (vm, s->next_buffer);
      s->ref_count = n_buffers;
    }

  return n_buffers;
}

/** \brief Create multiple clones of buffer and store them
    in the supplied array

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param src_buffer - (u32) source buffer index
    @param buffers - (u32 * ) buffer index array
    @param n_buffers - (u16) number of buffer clones requested (<=256)
    @param head_end_offset - (u16) offset relative to current position
           where packet head ends
    @param offset - (i16) copy packet head at current position if 0,
           else at offset position to change headroom space as specified
    @return - (u16) number of buffers actually cloned, may be
    less than the number requested or zero
*/
always_inline u16
vlib_buffer_clone_at_offset (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
			     u16 n_buffers, u16 head_end_offset, i16 offset)
{
  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
  u16 n_cloned = 0;

  while (n_buffers > 256)
    {
      vlib_buffer_t *copy;
      copy = vlib_buffer_copy (vm, s);
      n_cloned += vlib_buffer_clone_256 (vm,
					 vlib_get_buffer_index (vm, copy),
					 (buffers + n_cloned),
					 256, head_end_offset, offset);
      n_buffers -= 256;
    }
  n_cloned += vlib_buffer_clone_256 (vm, src_buffer,
				     buffers + n_cloned,
				     n_buffers, head_end_offset, offset);

  return n_cloned;
}

/** \brief Create multiple clones of buffer and store them
    in the supplied array

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param src_buffer - (u32) source buffer index
    @param buffers - (u32 * ) buffer index array
    @param n_buffers - (u16) number of buffer clones requested (<=256)
    @param head_end_offset - (u16) offset relative to current position
           where packet head ends
    @return - (u16) number of buffers actually cloned, may be
    less than the number requested or zero
*/
always_inline u16
vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
		   u16 n_buffers, u16 head_end_offset)
{
  return vlib_buffer_clone_at_offset (vm, src_buffer, buffers, n_buffers,
				      head_end_offset, 0);
}

/** \brief Attach cloned tail to the buffer

    @param vm - (vlib_main_t *) vlib main data structure pointer
    @param head - (vlib_buffer_t *) head buffer
    @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
*/

always_inline void
vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head,
			  vlib_buffer_t * tail)
{
  ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
  ASSERT (head->buffer_pool_index == tail->buffer_pool_index);

  head->flags |= VLIB_BUFFER_NEXT_PRESENT;
  head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
  head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
  head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
  head->next_buffer = vlib_get_buffer_index (vm, tail);
  head->total_length_not_including_first_buffer = tail->current_length +
    tail->total_length_not_including_first_buffer;

next_segment:
  clib_atomic_add_fetch (&tail->ref_count, 1);

  if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
    {
      tail = vlib_get_buffer (vm, tail->next_buffer);
      goto next_segment;
    }
}

/* Initializes the buffer as an empty packet with no chained buffers. */
always_inline void
vlib_buffer_chain_init (vlib_buffer_t * first)
{
  first->total_length_not_including_first_buffer = 0;
  first->current_length = 0;
  first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
  first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
}

/* The provided next_bi buffer index is appended to the end of the packet. */
always_inline vlib_buffer_t *
vlib_buffer_chain_buffer (vlib_main_t * vm, vlib_buffer_t * last, u32 next_bi)
{
  vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
  last->next_buffer = next_bi;
  last->flags |= VLIB_BUFFER_NEXT_PRESENT;
  next_buffer->current_length = 0;
  next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
  return next_buffer;
}

/* Increases or decreases the packet length.
 * It does not allocate or deallocate new buffers.
 * Therefore, the added length must be compatible
 * with the last buffer. */
always_inline void
vlib_buffer_chain_increase_length (vlib_buffer_t * first,
				   vlib_buffer_t * last, i32 len)
{
  last->current_length += len;
  if (first != last)
    first->total_length_not_including_first_buffer += len;
}

/* Copy data to the end of the packet and increases its length.
 * It does not allocate new buffers.
 * Returns the number of copied bytes. */
always_inline u16
vlib_buffer_chain_append_data (vlib_main_t * vm,
			       vlib_buffer_t * first,
			       vlib_buffer_t * last, void *data, u16 data_len)
{
  u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
  ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
  u16 len = clib_min (data_len,
		      n_buffer_bytes - last->current_length -
		      last->current_data);
  clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length,
		    data, len);
  vlib_buffer_chain_increase_length (first, last, len);
  return len;
}

/* Copy data to the end of the packet and increases its length.
 * Allocates additional buffers from the free list if necessary.
 * Returns the number of copied bytes.
 * 'last' value is modified whenever new buffers are allocated and
 * chained and points to the last buffer in the chain. */
u16
vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
					  vlib_buffer_t * first,
					  vlib_buffer_t ** last, void *data,
					  u16 data_len);
void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * first);

format_function_t format_vlib_buffer, format_vlib_buffer_and_data,
  format_vlib_buffer_contents;

typedef struct
{
  /* Vector of packet data. */
  u8 *packet_data;

  /* Number of buffers to allocate in each call to allocator. */
  u32 min_n_buffers_each_alloc;

  u8 *name;
} vlib_packet_template_t;

void vlib_packet_template_init (vlib_main_t * vm,
				vlib_packet_template_t * t,
				void *packet_data,
				uword n_packet_data_bytes,
				uword min_n_buffers_each_alloc,
				char *fmt, ...);

void *vlib_packet_template_get_packet (vlib_main_t * vm,
				       vlib_packet_template_t * t,
				       u32 * bi_result);

always_inline void
vlib_packet_template_free (vlib_main_t * vm, vlib_packet_template_t * t)
{
  vec_free (t->packet_data);
}

always_inline u32
vlib_buffer_space_left_at_end (vlib_main_t * vm, vlib_buffer_t * b)
{
  return b->data + vlib_buffer_get_default_data_size (vm) -
    ((u8 *) vlib_buffer_get_current (b) + b->current_length);
}

always_inline u32
vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * b)
{
  vlib_buffer_t *db = b, *sb, *first = b;
  int is_cloned = 0;
  u32 bytes_left = 0, data_size;
  u16 src_left, dst_left, n_buffers = 1;
  u8 *dp, *sp;
  u32 to_free = 0;

  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
    return 1;

  data_size = vlib_buffer_get_default_data_size (vm);

  dst_left = vlib_buffer_space_left_at_end (vm, b);

  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
    {
      b = vlib_get_buffer (vm, b->next_buffer);
      if (b->ref_count > 1)
	is_cloned = 1;
      bytes_left += b->current_length;
      n_buffers++;
    }

  /* if buffer is cloned, create completely new chain - unless everything fits
   * into one buffer */
  if (is_cloned && bytes_left >= dst_left)
    {
      u32 len = 0;
      u32 space_needed = bytes_left - dst_left;
      u32 tail;

      if (vlib_buffer_alloc (vm, &tail, 1) == 0)
	return 0;

      ++n_buffers;
      len += data_size;
      b = vlib_get_buffer (vm, tail);

      while (len < space_needed)
	{
	  u32 bi;
	  if (vlib_buffer_alloc (vm, &bi, 1) == 0)
	    {
	      vlib_buffer_free_one (vm, tail);
	      return 0;
	    }
	  b->flags = VLIB_BUFFER_NEXT_PRESENT;
	  b->next_buffer = bi;
	  b = vlib_get_buffer (vm, bi);
	  len += data_size;
	  n_buffers++;
	}
      sb = vlib_get_buffer (vm, first->next_buffer);
      to_free = first->next_buffer;
      first->next_buffer = tail;
    }
  else
    sb = vlib_get_buffer (vm, first->next_buffer);

  src_left = sb->current_length;
  sp = vlib_buffer_get_current (sb);
  dp = vlib_buffer_get_tail (db);

  while (bytes_left)
    {
      u16 bytes_to_copy;

      if (dst_left == 0)
	{
	  db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
	  ASSERT (db->flags & VLIB_BUFFER_NEXT_PRESENT);
	  db = vlib_get_buffer (vm, db->next_buffer);
	  dst_left = data_size;
	  if (db->current_data > 0)
	    {
	      db->current_data = 0;
	    }
	  else
	    {
	      dst_left += -db->current_data;
	    }
	  dp = vlib_buffer_get_current (db);
	}

      while (src_left == 0)
	{
	  ASSERT (sb->flags & VLIB_BUFFER_NEXT_PRESENT);
	  sb = vlib_get_buffer (vm, sb->next_buffer);
	  src_left = sb->current_length;
	  sp = vlib_buffer_get_current (sb);
	}

      bytes_to_copy = clib_min (dst_left, src_left);

      if (dp != sp)
	{
	  if (sb == db)
	    bytes_to_copy = clib_min (bytes_to_copy, sp - dp);

	  clib_memcpy_fast (dp, sp, bytes_to_copy);
	}

      src_left -= bytes_to_copy;
      dst_left -= bytes_to_copy;
      dp += bytes_to_copy;
      sp += bytes_to_copy;
      bytes_left -= bytes_to_copy;
    }
  if (db != first)
    db->current_data = 0;
  db->current_length = dp - (u8 *) vlib_buffer_get_current (db);

  if (is_cloned && to_free)
    vlib_buffer_free_one (vm, to_free);
  else
    {
      if (db->flags & VLIB_BUFFER_NEXT_PRESENT)
	vlib_buffer_free_one (vm, db->next_buffer);
      db->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
      b = first;
      n_buffers = 1;
      while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
	{
	  b = vlib_get_buffer (vm, b->next_buffer);
	  ++n_buffers;
	}
    }

  first->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;

  return n_buffers;
}

#endif /* included_vlib_buffer_funcs_h */

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */