From a638c18375f17891a9455e717d8436158af12387 Mon Sep 17 00:00:00 2001 From: Dave Barach Date: Fri, 21 Jun 2019 18:24:07 -0400 Subject: vlib: packet tracer support for pkt thread handoffs Type: feature Change-Id: Ia3d9a47679202c2a47cd3746b50e86c6b8627ef6 Signed-off-by: Dave Barach --- src/examples/handoffdemo/CMakeLists.txt | 20 +++ src/examples/handoffdemo/README.md | 186 ++++++++++++++++++++++++ src/examples/handoffdemo/handoffdemo.c | 39 ++++++ src/examples/handoffdemo/handoffdemo.h | 50 +++++++ src/examples/handoffdemo/node.c | 241 ++++++++++++++++++++++++++++++++ 5 files changed, 536 insertions(+) create mode 100644 src/examples/handoffdemo/CMakeLists.txt create mode 100644 src/examples/handoffdemo/README.md create mode 100644 src/examples/handoffdemo/handoffdemo.c create mode 100644 src/examples/handoffdemo/handoffdemo.h create mode 100644 src/examples/handoffdemo/node.c (limited to 'src/examples') diff --git a/src/examples/handoffdemo/CMakeLists.txt b/src/examples/handoffdemo/CMakeLists.txt new file mode 100644 index 00000000000..12bda6cffff --- /dev/null +++ b/src/examples/handoffdemo/CMakeLists.txt @@ -0,0 +1,20 @@ + +# Copyright (c) +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_vpp_plugin(handoffdemo + SOURCES + handoffdemo.c + node.c + handoffdemo.h +) diff --git a/src/examples/handoffdemo/README.md b/src/examples/handoffdemo/README.md new file mode 100644 index 00000000000..3787731c73f --- /dev/null +++ b/src/examples/handoffdemo/README.md @@ -0,0 +1,186 @@ +# Handoff queue demo plugin + +This plugin provides a simplified example of how to hand off +packets between threads. I used it to debug packet-tracer handoff +tracing support. + +# Packet generator input script + +``` + packet-generator new { + name x + limit 5 + size 128-128 + interface local0 + node handoffdemo-1 + data { + incrementing 30 + } + } +``` +# Start vpp with 2 worker threads + +The demo plugin hands packets from worker 1 to worker 2. + +# Enable tracing, and start the packet generator + +``` + trace add pg-input 100 + packet-generator enable +``` + +# Sample Run + +``` + DBGvpp# ex /tmp/pg_input_script + DBGvpp# pa en + DBGvpp# sh err + Count Node Reason + 5 handoffdemo-1 packets handed off processed + 5 handoffdemo-2 completed packets + DBGvpp# show run + Thread 1 vpp_wk_0 (lcore 0) + Time 133.9, average vectors/node 5.00, last 128 main loops 0.00 per node 0.00 + vector rates in 3.7331e-2, out 0.0000e0, drop 0.0000e0, punt 0.0000e0 + Name State Calls Vectors Suspends Clocks Vectors/Call + handoffdemo-1 active 1 5 0 4.76e3 5.00 + pg-input disabled 2 5 0 5.58e4 2.50 + unix-epoll-input polling 22760 0 0 2.14e7 0.00 + --------------- + Thread 2 vpp_wk_1 (lcore 2) + Time 133.9, average vectors/node 5.00, last 128 main loops 0.00 per node 0.00 + vector rates in 0.0000e0, out 0.0000e0, drop 3.7331e-2, punt 0.0000e0 + Name State Calls Vectors Suspends Clocks Vectors/Call + drop active 1 5 0 1.35e4 5.00 + error-drop active 1 5 0 2.52e4 5.00 + handoffdemo-2 active 1 5 0 2.56e4 5.00 + unix-epoll-input polling 22406 0 0 2.18e7 0.00 +``` + +Enable the packet tracer and run it again... + +``` + DBGvpp# trace add pg-input 100 + DBGvpp# pa en + DBGvpp# sh trace + sh trace + ------------------- Start of thread 0 vpp_main ------------------- + No packets in trace buffer + ------------------- Start of thread 1 vpp_wk_0 ------------------- + Packet 1 + + 00:06:50:520688: pg-input + stream x, 128 bytes, 0 sw_if_index + current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000000 + 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000 + 00000020: 0000000000000000000000000000000000000000000000000000000000000000 + 00000040: 0000000000000000000000000000000000000000000000000000000000000000 + 00000060: 0000000000000000000000000000000000000000000000000000000000000000 + 00:06:50:520762: handoffdemo-1 + HANDOFFDEMO: current thread 1 + + Packet 2 + + 00:06:50:520688: pg-input + stream x, 128 bytes, 0 sw_if_index + current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000001 + 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000 + 00000020: 0000000000000000000000000000000000000000000000000000000000000000 + 00000040: 0000000000000000000000000000000000000000000000000000000000000000 + 00000060: 0000000000000000000000000000000000000000000000000000000000000000 + 00:06:50:520762: handoffdemo-1 + HANDOFFDEMO: current thread 1 + + Packet 3 + + 00:06:50:520688: pg-input + stream x, 128 bytes, 0 sw_if_index + current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000002 + 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000 + 00000020: 0000000000000000000000000000000000000000000000000000000000000000 + 00000040: 0000000000000000000000000000000000000000000000000000000000000000 + 00000060: 0000000000000000000000000000000000000000000000000000000000000000 + 00:06:50:520762: handoffdemo-1 + HANDOFFDEMO: current thread 1 + + Packet 4 + + 00:06:50:520688: pg-input + stream x, 128 bytes, 0 sw_if_index + current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000003 + 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000 + 00000020: 0000000000000000000000000000000000000000000000000000000000000000 + 00000040: 0000000000000000000000000000000000000000000000000000000000000000 + 00000060: 0000000000000000000000000000000000000000000000000000000000000000 + 00:06:50:520762: handoffdemo-1 + HANDOFFDEMO: current thread 1 + + Packet 5 + + 00:06:50:520688: pg-input + stream x, 128 bytes, 0 sw_if_index + current data 0, length 128, buffer-pool 0, ref-count 1, trace handle 0x1000004 + 00000000: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d0000 + 00000020: 0000000000000000000000000000000000000000000000000000000000000000 + 00000040: 0000000000000000000000000000000000000000000000000000000000000000 + 00000060: 0000000000000000000000000000000000000000000000000000000000000000 + 00:06:50:520762: handoffdemo-1 + HANDOFFDEMO: current thread 1 + + ------------------- Start of thread 2 vpp_wk_1 ------------------- + Packet 1 + + 00:06:50:520796: handoff_trace + HANDED-OFF: from thread 1 trace index 0 + 00:06:50:520796: handoffdemo-2 + HANDOFFDEMO: current thread 2 + 00:06:50:520867: error-drop + rx:local0 + 00:06:50:520914: drop + handoffdemo-2: completed packets + + Packet 2 + + 00:06:50:520796: handoff_trace + HANDED-OFF: from thread 1 trace index 1 + 00:06:50:520796: handoffdemo-2 + HANDOFFDEMO: current thread 2 + 00:06:50:520867: error-drop + rx:local0 + 00:06:50:520914: drop + handoffdemo-2: completed packets + + Packet 3 + + 00:06:50:520796: handoff_trace + HANDED-OFF: from thread 1 trace index 2 + 00:06:50:520796: handoffdemo-2 + HANDOFFDEMO: current thread 2 + 00:06:50:520867: error-drop + rx:local0 + 00:06:50:520914: drop + handoffdemo-2: completed packets + + Packet 4 + + 00:06:50:520796: handoff_trace + HANDED-OFF: from thread 1 trace index 3 + 00:06:50:520796: handoffdemo-2 + HANDOFFDEMO: current thread 2 + 00:06:50:520867: error-drop + rx:local0 + 00:06:50:520914: drop + handoffdemo-2: completed packets + + Packet 5 + + 00:06:50:520796: handoff_trace + HANDED-OFF: from thread 1 trace index 4 + 00:06:50:520796: handoffdemo-2 + HANDOFFDEMO: current thread 2 + 00:06:50:520867: error-drop + rx:local0 + 00:06:50:520914: drop + handoffdemo-2: completed packets + DBGvpp# +``` diff --git a/src/examples/handoffdemo/handoffdemo.c b/src/examples/handoffdemo/handoffdemo.c new file mode 100644 index 00000000000..a1dad44d28d --- /dev/null +++ b/src/examples/handoffdemo/handoffdemo.c @@ -0,0 +1,39 @@ +/* + * handoffdemo.c - skeleton vpp engine plug-in + * + * Copyright (c) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +handoffdemo_main_t handoffdemo_main; + +/* *INDENT-OFF* */ +VLIB_PLUGIN_REGISTER () = +{ + .version = VPP_BUILD_VER, + .description = "handoff demo plugin", +}; +/* *INDENT-ON* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/examples/handoffdemo/handoffdemo.h b/src/examples/handoffdemo/handoffdemo.h new file mode 100644 index 00000000000..558311fe29e --- /dev/null +++ b/src/examples/handoffdemo/handoffdemo.h @@ -0,0 +1,50 @@ + +/* + * handoffdemo.h - skeleton vpp engine plug-in header file + * + * Copyright (c) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_handoffdemo_h__ +#define __included_handoffdemo_h__ + +#include +#include +#include + +#include +#include + +typedef struct +{ + u32 frame_queue_index; + + + /* convenience */ + vlib_main_t *vlib_main; + vnet_main_t *vnet_main; +} handoffdemo_main_t; + +extern handoffdemo_main_t handoffdemo_main; + +extern vlib_node_registration_t handoffdemo_node_1, handoffdemo_node_2; + +#endif /* __included_handoffdemo_h__ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/examples/handoffdemo/node.c b/src/examples/handoffdemo/node.c new file mode 100644 index 00000000000..2177ccb9d14 --- /dev/null +++ b/src/examples/handoffdemo/node.c @@ -0,0 +1,241 @@ +/* + * node.c - skeleton vpp engine plug-in dual-loop node skeleton + * + * Copyright (c) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include + +typedef struct +{ + int current_thread; +} handoffdemo_trace_t; + +/* packet trace format function */ +static u8 * +format_handoffdemo_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + handoffdemo_trace_t *t = va_arg (*args, handoffdemo_trace_t *); + + s = format (s, "HANDOFFDEMO: current thread %d", t->current_thread); + + return s; +} + +vlib_node_registration_t handoffdemo_node; + +#define foreach_handoffdemo_error \ +_(HANDED_OFF, "packets handed off processed") \ +_(CONGESTION_DROP, "handoff queue congestion drops") \ +_(COMPLETE, "completed packets") + +typedef enum +{ +#define _(sym,str) HANDOFFDEMO_ERROR_##sym, + foreach_handoffdemo_error +#undef _ + HANDOFFDEMO_N_ERROR, +} handoffdemo_error_t; + +static char *handoffdemo_error_strings[] = { +#define _(sym,string) string, + foreach_handoffdemo_error +#undef _ +}; + +typedef enum +{ + HANDOFFDEMO_NEXT_DROP, + HANDOFFDEMO_N_NEXT, +} handoffdemo_next_t; + +always_inline uword +handoffdemo_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame, + int which, int is_trace) +{ + handoffdemo_main_t *hmp = &handoffdemo_main; + u32 n_left_from, *from; + u32 error0 = node->errors[HANDOFFDEMO_ERROR_COMPLETE]; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; + u16 thread_indices[VLIB_FRAME_SIZE]; + u16 nexts[VLIB_FRAME_SIZE], *next; + u32 n_enq; + int i; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + + vlib_get_buffers (vm, from, bufs, n_left_from); + next = nexts; + b = bufs; + + /* First thread */ + if (which == 1) + { + for (i = 0; i < frame->n_vectors; i++) + { + /* Pick a thread to handle this packet */ + thread_indices[i] = 2; + + if (is_trace && (b[0]->flags & VLIB_BUFFER_IS_TRACED)) + { + handoffdemo_trace_t *t = vlib_add_trace (vm, node, b[0], + sizeof (*t)); + t->current_thread = vm->thread_index; + } + + b += 1; + next += 1; + n_left_from -= 1; + } + + /* Enqueue buffers to threads */ + n_enq = + vlib_buffer_enqueue_to_thread (vm, hmp->frame_queue_index, + from, thread_indices, frame->n_vectors, + 1 /* drop on congestion */ ); + if (n_enq < frame->n_vectors) + vlib_node_increment_counter (vm, node->node_index, + HANDOFFDEMO_ERROR_CONGESTION_DROP, + frame->n_vectors - n_enq); + vlib_node_increment_counter (vm, node->node_index, + HANDOFFDEMO_ERROR_HANDED_OFF, n_enq); + return frame->n_vectors; + } + else /* Second thread */ + { + u32 *from; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + + vlib_get_buffers (vm, from, bufs, n_left_from); + next = nexts; + b = bufs; + + while (n_left_from > 0) + { + if (is_trace && (b[0]->flags & VLIB_BUFFER_IS_TRACED)) + { + handoffdemo_trace_t *t = vlib_add_trace (vm, node, b[0], + sizeof (*t)); + t->current_thread = vm->thread_index; + } + + next[0] = HANDOFFDEMO_NEXT_DROP; + b[0]->error = error0; + next++; + b++; + n_left_from--; + } + + vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts, + frame->n_vectors); + } + + return frame->n_vectors; +} + +static uword +handoffdemo_node_1_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) + return handoffdemo_inline (vm, node, frame, 1 /* which */ , + 1 /* is_trace */ ); + else + return handoffdemo_inline (vm, node, frame, 1 /* which */ , + 0 /* is_trace */ ); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (handoffdemo_node_1) = +{ + .name = "handoffdemo-1", + .function = handoffdemo_node_1_fn, + .vector_size = sizeof (u32), + .format_trace = format_handoffdemo_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(handoffdemo_error_strings), + .error_strings = handoffdemo_error_strings, + + .n_next_nodes = HANDOFFDEMO_N_NEXT, + + /* edit / add dispositions here */ + .next_nodes = { + [HANDOFFDEMO_NEXT_DROP] = "error-drop", + }, +}; +/* *INDENT-ON* */ + +uword +handoffdemo_node_2_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) + return handoffdemo_inline (vm, node, frame, 2 /* which */ , + 1 /* is_trace */ ); + else + return handoffdemo_inline (vm, node, frame, 2 /* which */ , + 0 /* is_trace */ ); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (handoffdemo_node_2) = +{ + .name = "handoffdemo-2", + .function = handoffdemo_node_2_fn, + .vector_size = sizeof (u32), + .format_trace = format_handoffdemo_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(handoffdemo_error_strings), + .error_strings = handoffdemo_error_strings, + + .n_next_nodes = HANDOFFDEMO_N_NEXT, + + /* edit / add dispositions here */ + .next_nodes = { + [HANDOFFDEMO_NEXT_DROP] = "error-drop", + }, +}; +/* *INDENT-ON* */ + +static clib_error_t * +handoffdemo_node_init (vlib_main_t * vm) +{ + handoffdemo_main_t *hmp = &handoffdemo_main; + + hmp->frame_queue_index = vlib_frame_queue_main_init + (handoffdemo_node_2.index, 16); + + return 0; +} + +VLIB_INIT_FUNCTION (handoffdemo_node_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ -- cgit 1.2.3-korg