aboutsummaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/plugins/dpdk/buffer.c69
-rw-r--r--src/vlib/buffer.c57
-rw-r--r--src/vlib/buffer.h32
-rw-r--r--src/vlib/buffer_funcs.h13
-rw-r--r--src/vlib/main.c20
-rw-r--r--src/vlib/unix/physmem.c9
-rw-r--r--src/vnet/pg/input.c6
-rw-r--r--src/vnet/pg/stream.c2
-rw-r--r--src/vnet/replication.c2
9 files changed, 96 insertions, 114 deletions
diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c
index fd1d84153db..aa73eb6c008 100644
--- a/src/plugins/dpdk/buffer.c
+++ b/src/plugins/dpdk/buffer.c
@@ -431,7 +431,6 @@ vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
unsigned socket_id)
{
dpdk_main_t *dm = &dpdk_main;
- vlib_physmem_main_t *vpm = &vm->physmem_main;
struct rte_mempool *rmp;
int i;
@@ -453,63 +452,10 @@ vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
if (rmp)
{
{
- uword this_pool_end;
- uword this_pool_start;
- uword this_pool_size;
- uword save_vpm_start, save_vpm_end, save_vpm_size;
struct rte_mempool_memhdr *memhdr;
- this_pool_start = ~0;
- this_pool_end = 0;
-
STAILQ_FOREACH (memhdr, &rmp->mem_list, next)
- {
- if (((uword) (memhdr->addr + memhdr->len)) > this_pool_end)
- this_pool_end = (uword) (memhdr->addr + memhdr->len);
- if (((uword) memhdr->addr) < this_pool_start)
- this_pool_start = (uword) (memhdr->addr);
- }
- ASSERT (this_pool_start < ~0 && this_pool_end > 0);
- this_pool_size = this_pool_end - this_pool_start;
-
- if (CLIB_DEBUG > 1)
- {
- clib_warning ("%s: pool start %llx pool end %llx pool size %lld",
- pool_name, this_pool_start, this_pool_end,
- this_pool_size);
- clib_warning
- ("before: virtual.start %llx virtual.end %llx virtual.size %lld",
- vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
- }
-
- save_vpm_start = vpm->virtual.start;
- save_vpm_end = vpm->virtual.end;
- save_vpm_size = vpm->virtual.size;
-
- if ((this_pool_start < vpm->virtual.start) || vpm->virtual.start == 0)
- vpm->virtual.start = this_pool_start;
- if (this_pool_end > vpm->virtual.end)
- vpm->virtual.end = this_pool_end;
-
- vpm->virtual.size = vpm->virtual.end - vpm->virtual.start;
-
- if (CLIB_DEBUG > 1)
- {
- clib_warning
- ("after: virtual.start %llx virtual.end %llx virtual.size %lld",
- vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
- }
-
- /* check if fits into buffer index range */
- if ((u64) vpm->virtual.size >
- ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
- {
- clib_warning ("physmem: virtual size out of range!");
- vpm->virtual.start = save_vpm_start;
- vpm->virtual.end = save_vpm_end;
- vpm->virtual.size = save_vpm_size;
- rmp = 0;
- }
+ vlib_buffer_add_mem_range (vm, (uword) memhdr->addr, memhdr->len);
}
if (rmp)
{
@@ -564,7 +510,8 @@ buffer_state_validation_init (vlib_main_t * vm)
VLIB_INIT_FUNCTION (buffer_state_validation_init);
#endif
-static vlib_buffer_callbacks_t callbacks = {
+/* *INDENT-OFF* */
+VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
.vlib_buffer_alloc_cb = &dpdk_buffer_alloc,
.vlib_buffer_alloc_from_free_list_cb = &dpdk_buffer_alloc_from_free_list,
.vlib_buffer_free_cb = &dpdk_buffer_free,
@@ -572,15 +519,7 @@ static vlib_buffer_callbacks_t callbacks = {
.vlib_packet_template_init_cb = &dpdk_packet_template_init,
.vlib_buffer_delete_free_list_cb = &dpdk_buffer_delete_free_list,
};
-
-static clib_error_t *
-dpdk_buffer_init (vlib_main_t * vm)
-{
- vlib_buffer_cb_register (vm, &callbacks);
- return 0;
-}
-
-VLIB_INIT_FUNCTION (dpdk_buffer_init);
+/* *INDENT-ON* */
/** @endcond */
/*
diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index e50064ae622..b2a095cfb1a 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -46,6 +46,8 @@
#include <vlib/vlib.h>
#include <vlib/unix/unix.h>
+vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0;
+
uword
vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
vlib_buffer_t * b_first)
@@ -584,11 +586,6 @@ alloc_from_free_list (vlib_main_t * vm,
dst = alloc_buffers;
- /* wait with buffer memory allocation as long as possible
- in case external buffer manager takes over */
- if (PREDICT_FALSE (vm->os_physmem_alloc_aligned == 0))
- unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ );
-
n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
if (n_filled == 0)
return 0;
@@ -944,6 +941,36 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
return copied;
}
+void
+vlib_buffer_add_mem_range (vlib_main_t * vm, uword start, uword size)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+
+ if (bm->buffer_mem_size == 0)
+ {
+ bm->buffer_mem_start = start;
+ bm->buffer_mem_size = size;
+ }
+ else if (start < bm->buffer_mem_start)
+ {
+ bm->buffer_mem_size += bm->buffer_mem_start - start;
+ bm->buffer_mem_start = start;
+ if (size > bm->buffer_mem_size)
+ bm->buffer_mem_size = size;
+ }
+ else if (start > bm->buffer_mem_start)
+ {
+ uword new_size = start - bm->buffer_mem_start + size;
+ if (new_size > bm->buffer_mem_size)
+ bm->buffer_mem_size = new_size;
+ }
+
+ if ((u64) bm->buffer_mem_size >
+ ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
+ {
+ clib_panic ("buffer memory size out of range!");
+ }
+}
static u8 *
format_vlib_buffer_free_list (u8 * s, va_list * va)
@@ -1011,6 +1038,7 @@ void
vlib_buffer_cb_init (struct vlib_main_t *vm)
{
vlib_buffer_main_t *bm = vm->buffer_main;
+
bm->cb.vlib_buffer_alloc_cb = &vlib_buffer_alloc_internal;
bm->cb.vlib_buffer_alloc_from_free_list_cb =
&vlib_buffer_alloc_from_free_list_internal;
@@ -1018,25 +1046,6 @@ vlib_buffer_cb_init (struct vlib_main_t *vm)
bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal;
bm->cb.vlib_buffer_delete_free_list_cb =
&vlib_buffer_delete_free_list_internal;
- bm->extern_buffer_mgmt = 0;
-}
-
-int
-vlib_buffer_cb_register (struct vlib_main_t *vm, vlib_buffer_callbacks_t * cb)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- if (bm->extern_buffer_mgmt)
- return -1;
-
-#define _(x) bm->cb.x = cb->x
- _(vlib_buffer_alloc_cb);
- _(vlib_buffer_alloc_from_free_list_cb);
- _(vlib_buffer_free_cb);
- _(vlib_buffer_free_no_next_cb);
- _(vlib_buffer_delete_free_list_cb);
-#undef _
- bm->extern_buffer_mgmt = 1;
- return 0;
}
/** @endcond */
diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h
index 18e2437d775..b20538b7f8c 100644
--- a/src/vlib/buffer.h
+++ b/src/vlib/buffer.h
@@ -388,12 +388,20 @@ typedef struct
u32 free_list_index);
} vlib_buffer_callbacks_t;
+extern vlib_buffer_callbacks_t *vlib_buffer_callbacks;
+
typedef struct
{
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+ /* Virtual memory address and size of buffer memory, used for calculating
+ buffer index */
+ uword buffer_mem_start;
+ uword buffer_mem_size;
+
/* Buffer free callback, for subversive activities */
- u32 (*buffer_free_callback) (struct vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers, u32 follow_buffer_next);
+ u32 (*buffer_free_callback) (struct vlib_main_t * vm,
+ u32 * buffers,
+ u32 n_buffers, u32 follow_buffer_next);
/* Pool of buffer free lists.
Multiple free lists exist for packet generator which uses
separate free lists for each packet stream --- so as to avoid
@@ -417,12 +425,12 @@ typedef struct
/* Callbacks */
vlib_buffer_callbacks_t cb;
- int extern_buffer_mgmt;
+ int callbacks_registered;
} vlib_buffer_main_t;
+void vlib_buffer_add_mem_range (struct vlib_main_t *vm, uword start,
+ uword size);
void vlib_buffer_cb_init (struct vlib_main_t *vm);
-int vlib_buffer_cb_register (struct vlib_main_t *vm,
- vlib_buffer_callbacks_t * cb);
typedef struct
{
@@ -498,6 +506,18 @@ serialize_vlib_buffer_n_bytes (serialize_main_t * m)
#endif /* included_vlib_buffer_h */
+#define VLIB_BUFFER_REGISTER_CALLBACKS(x,...) \
+ __VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks; \
+static void __vlib_add_buffer_callbacks_t_##x (void) \
+ __attribute__((__constructor__)) ; \
+static void __vlib_add_buffer_callbacks_t_##x (void) \
+{ \
+ if (vlib_buffer_callbacks) \
+ clib_panic ("vlib buffer callbacks already registered"); \
+ vlib_buffer_callbacks = &__##x##_buffer_callbacks; \
+} \
+__VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks
+
/*
* fd.io coding-style-patch-verification: ON
*
diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index 328660a3428..79e3e69c919 100644
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -56,8 +56,11 @@
always_inline vlib_buffer_t *
vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
{
- return vlib_physmem_at_offset (&vm->physmem_main, ((uword) buffer_index)
- << CLIB_LOG2_CACHE_LINE_BYTES);
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
+ ASSERT (offset < bm->buffer_mem_size);
+
+ return uword_to_pointer (bm->buffer_mem_start + offset, void *);
}
/** \brief Translate buffer pointer into buffer index
@@ -66,10 +69,14 @@ vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
@param p - (void *) buffer pointer
@return - (u32) buffer index
*/
+
always_inline u32
vlib_get_buffer_index (vlib_main_t * vm, void *p)
{
- uword offset = vlib_physmem_offset_of (&vm->physmem_main, p);
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
+ ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
+ ASSERT (offset < bm->buffer_mem_size);
ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
}
diff --git a/src/vlib/main.c b/src/vlib/main.c
index 19d70232e6e..73548fbea72 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -43,6 +43,7 @@
#include <vlib/threads.h>
#include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
+#include <vlib/unix/unix.h>
#include <vlib/unix/cj.h>
CJ_GLOBAL_LOG_PROTOTYPE;
@@ -466,7 +467,7 @@ vlib_put_next_frame (vlib_main_t * vm,
vlib_frame_t *f;
u32 n_vectors_in_frame;
- if (vm->buffer_main->extern_buffer_mgmt == 0 && CLIB_DEBUG > 0)
+ if (vm->buffer_main->callbacks_registered == 0 && CLIB_DEBUG > 0)
vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
@@ -1712,7 +1713,22 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
vm->name = "VLIB";
vec_validate (vm->buffer_main, 0);
- vlib_buffer_cb_init (vm);
+ if (vlib_buffer_callbacks)
+ {
+ /* external plugin has registered own buffer callbacks
+ so we just copy them */
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ clib_memcpy (&bm->cb, vlib_buffer_callbacks,
+ sizeof (vlib_buffer_callbacks_t));
+ bm->callbacks_registered = 1;
+ }
+ else
+ {
+ vlib_physmem_main_t *vpm = &vm->physmem_main;
+ vlib_buffer_cb_init (vm);
+ unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ );
+ vlib_buffer_add_mem_range (vm, vpm->virtual.start, vpm->virtual.size);
+ }
if ((error = vlib_thread_init (vm)))
{
diff --git a/src/vlib/unix/physmem.c b/src/vlib/unix/physmem.c
index 8d10ad2e88d..933cc63b7f4 100644
--- a/src/vlib/unix/physmem.c
+++ b/src/vlib/unix/physmem.c
@@ -45,14 +45,10 @@ static void *
unix_physmem_alloc_aligned (vlib_physmem_main_t * vpm, uword n_bytes,
uword alignment)
{
- vlib_main_t *vm = vlib_get_main ();
physmem_main_t *pm = &physmem_main;
uword lo_offset, hi_offset;
uword *to_free = 0;
- if (vm->buffer_main->extern_buffer_mgmt)
- clib_warning ("unsafe alloc!");
-
/* IO memory is always at least cache aligned. */
alignment = clib_max (alignment, CLIB_CACHE_LINE_BYTES);
@@ -270,11 +266,6 @@ show_physmem (vlib_main_t * vm,
unformat_input_t * input, vlib_cli_command_t * cmd)
{
physmem_main_t *pm = &physmem_main;
- if (vm->buffer_main->extern_buffer_mgmt)
- {
- vlib_cli_output (vm, "Not supported with external buffer management.");
- return 0;
- }
if (pm->heap)
vlib_cli_output (vm, "%U", format_mheap, pm->heap, /* verbose */ 1);
diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c
index 597ae060045..c3738a6a2aa 100644
--- a/src/vnet/pg/input.c
+++ b/src/vnet/pg/input.c
@@ -1214,7 +1214,7 @@ pg_stream_fill_helper (pg_main_t * pg,
* Historically, the pg maintained its own free lists and
* device drivers tx paths would return pkts.
*/
- if (vm->buffer_main->extern_buffer_mgmt == 0 &&
+ if (vm->buffer_main->callbacks_registered == 0 &&
!(s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE))
f->buffer_init_function = pg_buffer_init;
f->buffer_init_function_opaque =
@@ -1238,7 +1238,7 @@ pg_stream_fill_helper (pg_main_t * pg,
n_alloc = n_allocated;
/* Reinitialize buffers */
- if (vm->buffer_main->extern_buffer_mgmt == 0 || CLIB_DEBUG > 0
+ if (vm->buffer_main->callbacks_registered == 0 || CLIB_DEBUG > 0
|| (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE))
init_buffers_inline
(vm, s,
@@ -1246,7 +1246,7 @@ pg_stream_fill_helper (pg_main_t * pg,
n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ ,
s->buffer_bytes,
/* set_data */
- vm->buffer_main->extern_buffer_mgmt != 0
+ vm->buffer_main->callbacks_registered != 0
|| (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE) != 0);
if (next_buffers)
diff --git a/src/vnet/pg/stream.c b/src/vnet/pg/stream.c
index 05d820a3807..a540b32bba4 100644
--- a/src/vnet/pg/stream.c
+++ b/src/vnet/pg/stream.c
@@ -438,7 +438,7 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
pg_buffer_index_t *bi;
int n;
- if (vm->buffer_main->extern_buffer_mgmt)
+ if (vm->buffer_main->callbacks_registered)
s->buffer_bytes = VLIB_BUFFER_DATA_SIZE;
if (!s->buffer_bytes)
diff --git a/src/vnet/replication.c b/src/vnet/replication.c
index 233a8c2f100..1c6f28d2493 100644
--- a/src/vnet/replication.c
+++ b/src/vnet/replication.c
@@ -214,7 +214,7 @@ replication_recycle_callback (vlib_main_t * vm, vlib_buffer_free_list_t * fl)
b0->flags |= VLIB_BUFFER_IS_RECYCLED;
#if (CLIB_DEBUG > 0)
- if (vm->buffer_main->extern_buffer_mgmt == 0)
+ if (vm->buffer_main->callbacks_registered == 0)
vlib_buffer_set_known_state (vm, bi0,
VLIB_BUFFER_KNOWN_ALLOCATED);
#endif