/* * Copyright (c) 2015 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * buffer.c: allocate/free network buffers. * * Copyright (c) 2008 Eliot Dresselhaus * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /** * @file * * Allocate/free network buffers. */ #include #include #include #include #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA 16384 #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV 8192 #ifdef CLIB_HAVE_VEC128 /* Assumptions by vlib_buffer_free_inline: */ STATIC_ASSERT_FITS_IN (vlib_buffer_t, flags, 16); STATIC_ASSERT_FITS_IN (vlib_buffer_t, ref_count, 16); STATIC_ASSERT_FITS_IN (vlib_buffer_t, buffer_pool_index, 16); #endif /* Make sure that buffer template size is not accidentally changed */ STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64); u16 __vlib_buffer_external_hdr_size = 0; static void buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e, u32 index); static void buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e, u32 index); static void buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index); uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, vlib_buffer_t * b_first) { vlib_buffer_t *b = b_first; uword l_first = b_first->current_length; uword l = 0; while (b->flags & VLIB_BUFFER_NEXT_PRESENT) { b = vlib_get_buffer (vm, b->next_buffer); l += b->current_length; } b_first->total_length_not_including_first_buffer = l; b_first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; return l + l_first; } u8 * format_vlib_buffer_no_chain (u8 * s, va_list * args) { vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); u32 indent = format_get_indent (s); u8 *a = 0; #define _(bit, name, v) \ if (v && (b->flags & VLIB_BUFFER_##name)) \ a = format (a, "%s ", v); foreach_vlib_buffer_flag #undef _ s = format (s, "current data %d, length %d, buffer-pool %d, " "ref-count %u", b->current_data, b->current_length, b->buffer_pool_index, b->ref_count); if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID) s = format (s, ", totlen-nifb %d", b->total_length_not_including_first_buffer); if (b->flags & VLIB_BUFFER_IS_TRACED) s = format (s, ", trace handle 0x%x", b->trace_handle); if (a) s = format (s, "\n%U%v", format_white_space, indent, a); vec_free (a); return s; } u8 * format_vlib_buffer (u8 * s, va_list * args) { vlib_main_t *vm = vlib_get_main (); vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); u32 indent = format_get_indent (s); s = format (s, "%U", format_vlib_buffer_no_chain, b); while (b->flags & VLIB_BUFFER_NEXT_PRESENT) { u32 next_buffer = b->next_buffer; b = vlib_get_buffer (vm, next_buffer); s = format (s, "\n%Unext-buffer 0x%x, segment length %d, ref-count %u", format_white_space, indent, next_buffer, b->current_length, b->ref_count); } return s; } u8 * format_vlib_buffer_and_data (u8 * s, va_list * args) { vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); s = format (s, "%U, %U", format_vlib_buffer, b, format_hex_bytes, vlib_buffer_get_current (b), 64); return s; } static u8 * format_vlib_buffer_known_state (u8 * s, va_list * args) { vlib_buffer_known_state_t state = va_arg (*args, vlib_buffer_known_state_t); char *t; switch (state) { case VLIB_BUFFER_UNKNOWN: t = "unknown"; break; case VLIB_BUFFER_KNOWN_ALLOCATED: t = "known-allocated"; break; case VLIB_BUFFER_KNOWN_FREE: t = "known-free"; break; default: t = "invalid"; break; } return format (s, "%s", t); } u8 * format_vlib_buffer_contents (u8 * s, va_list * va) { vlib_main_t *vm = va_arg (*va, vlib_main_t *); vlib_buffer_t *b = va_arg (*va, vlib_buffer_t *); while (1) { vec_add (s, vlib_buffer_get_current (b), b->current_length); if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)) break; b = vlib_get_buffer (vm, b->next_buffer); } return s; } static u8 * vlib_validate_buffer_helper (vlib_main_t * vm, u32 bi, uword follow_buffer_next, uword ** unique_hash) { vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_t *b = vlib_get_buffer (vm, bi); if (vec_len (bm->buffer_pools) <= b->buffer_pool_index) return format (0, "unknown buffer pool 0x%x", b->buffer_pool_index); if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE) return format (0, "current data %d before pre-data", b->current_data); if (b->current_data + b->current_length > vlib_buffer_get_default_data_size (vm)) return format (0, "%d-%d beyond end of buffer %d", b->current_data, b->current_length, vlib_buffer_get_default_data_size (vm)); if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT)) { vlib_buffer_known_state_t k; u8 *msg, *result; k = vlib_buffer_is_known (vm, b->next_buffer); if (k != VLIB_BUFFER_KNOWN_ALLOCATED) return format (0, "next 0x%x: %U", b->next_buffer, format_vlib_buffer_known_state, k); if (unique_hash) { if (hash_get (*unique_hash, b->next_buffer)) return format (0, "duplicate buffer 0x%x", b->next_buffer); hash_set1 (*unique_hash, b->next_buffer); } msg = vlib_validate_buffer (vm, b->next_buffer, follow_buffer_next); if (msg) { result = format (0, "next 0x%x: %v", b->next_buffer, msg); vec_free (msg); return result; } } return 0; } u8 * vlib_validate_buffer (vlib_main_t * vm, u32 bi, uword follow_buffer_next) { return vlib_validate_buffer_helper (vm, bi, follow_buffer_next, /* unique_hash */ 0); } u8 * vlib_validate_buffers (vlib_main_t * vm, u32 * buffers, uword next_buffer_stride, uword n_buffers, vlib_buffer_known_state_t known_state, uword follow_buffer_next) { uword i, *hash; u32 bi, *b = buffers; vlib_buffer_known_state_t k; u8 *msg = 0, *result = 0; hash = hash_create (0, 0); for (i = 0; i < n_buffers; i++) { bi = b[0]; b += next_buffer_stride; /* Buffer is not unique. */ if (hash_get (hash, bi)) { msg = format (0, "not unique"); goto done; } k = vlib_buffer_is_known (vm, bi); if (k != known_state) { msg = format (0, "is %U; expected %U", format_vlib_buffer_known_state, k, format_vlib_buffer_known_state, known_state); goto done; } msg = vlib_validate_buffer_helper (vm, bi, follow_buffer_next, &hash); if (msg) goto done; hash_set1 (hash, bi); } done: if (msg) { result = format (0, "0x%x: %v", bi, msg); vec_free (msg); } hash_free (hash); return result; } /* * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1 * and vlib_mains[0] = &vlib_global_main from the beginning of time. * * The only place which should ever expand vlib_mains is start_workers() * in threads.c. It knows about the bootstrap vector. */ /* *INDENT-OFF* */ static struct { vec_header_t h; vlib_main_t *vm; } __attribute__ ((packed)) __bootstrap_vlib_main_vector __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) = { .h.len = 1, .vm = &vlib_global_main, }; /* *INDENT-ON* */ vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm; /* When debugging validate that given buffers are either known allocated or known free. */ void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers, uword n_buffers, vlib_buffer_known_state_t expected_state) { vlib_buffer_main_t *bm = vm->buffer_main; u32 *b; uword i, bi, is_free; if (CLIB_DEBUG == 0) return; is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED; b = buffers; for (i = 0; i < n_buff
/*
 *------------------------------------------------------------------
 * Copyright (c) 2018 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *------------------------------------------------------------------
 */

#ifndef SRC_VLIBMEMORY_MEMORY_SHARED_H_
#define SRC_VLIBMEMORY_MEMORY_SHARED_H_

#include <vlibapi/api_common.h>
#include <vppinfra/error.h>

/* Allocated in shared memory */

/*
 * Ring-allocation scheme for client API messages
 *
 * Only one proc/thread has control of a given message buffer.
 * To free a buffer allocated from one of these rings, we clear
 * a field in the buffer (header), and leave.
 *
 * No locks, no hits, no errors...
 */
typedef struct ring_alloc_
{
  svm_queue_t *rp;
  u16 size;
  u16 nitems;
  u32 hits;
  u32 misses;
} ring_alloc_t;

typedef enum
{
  VL_API_VLIB_RING,
  VL_API_CLIENT_RING,
  VL_API_QUEUE
} vl_api_shm_config_type_t;

typedef struct vl_api_shm_elem_config_
{
  u8 type;
  u8 _pad;
  u16 count;
  u32 size;
} vl_api_shm_elem_config_t;

STATIC_ASSERT (sizeof (vl_api_shm_elem_config_t) == 8,
	       "Size must be exactly 8 bytes");

/*
 * Initializers for the (shared-memory) rings
 * _(size, n). Note: each msg has space for a header.
 */
#define foreach_vl_aring_size                   \
_(64+sizeof(ring_alloc_t), 1024)                \
_(256+sizeof(ring_alloc_t), 128)                \
_(1024+sizeof(ring_alloc_t), 64)

#define foreach_clnt_aring_size                 \
 _(1024+sizeof(ring_alloc_t), 1024)             \
 _(2048+sizeof(ring_alloc_t), 128)              \
 _(4096+sizeof(ring_alloc_t), 8)

typedef struct vl_shmem_hdr_
{
  int version;

  /* getpid () for the VLIB client process */
  volatile int vl_pid;

  /* Client sends VLIB msgs here. */
  svm_queue_t *vl_input_queue;

  /* Vector of rings; one for each size. */

  /* VLIB allocates buffers to send msgs to clients here. */
  ring_alloc_t *vl_rings;

  /* Clients allocate buffer to send msgs to VLIB here. */
  ring_alloc_t *client_rings;

  /* Number of detected application restarts */
  u32 application_restarts;

  /* Number of messages reclaimed during application restart */
  u32 restart_reclaims;

  /* Number of garbage-collected messages */
  u32 garbage_collects;

  /* Socket file index used to bootstrap shmem region */
  u32 clib_file_index;
} vl_shmem_hdr_t;

#define VL_SHM_VERSION 2
#define VL_API_EPOCH_MASK 0xFF
#define VL_API_EPOCH_SHIFT 8

void *vl_msg_api_alloc (int