diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/vlib/node_funcs.h | 14 | ||||
| -rw-r--r-- | src/vnet/devices/af_packet/af_packet.c | 7 | ||||
| -rw-r--r-- | src/vnet/devices/af_packet/node.c | 2 | ||||
| -rw-r--r-- | src/vnet/devices/devices.c | 135 | ||||
| -rw-r--r-- | src/vnet/devices/devices.h | 60 | ||||
| -rw-r--r-- | src/vnet/interface.c | 2 | ||||
| -rw-r--r-- | src/vnet/interface.h | 3 |
7 files changed, 194 insertions, 29 deletions
diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index 54e36874..4d7cc192 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -177,6 +177,20 @@ vlib_node_set_state (vlib_main_t * vm, u32 node_index, r->state = new_state; } +/** \brief Get node dispatch state. + @param vm vlib_main_t pointer, varies by thread + @param node_index index of the node + @return state for node, see vlib_node_state_t +*/ +always_inline vlib_node_state_t +vlib_node_get_state (vlib_main_t * vm, u32 node_index) +{ + vlib_node_main_t *nm = &vm->node_main; + vlib_node_t *n; + n = vec_elt (nm->nodes, node_index); + return n->state; +} + always_inline void vlib_node_set_interrupt_pending (vlib_main_t * vm, u32 node_index) { diff --git a/src/vnet/devices/af_packet/af_packet.c b/src/vnet/devices/af_packet/af_packet.c index 20285107..7464d4e6 100644 --- a/src/vnet/devices/af_packet/af_packet.c +++ b/src/vnet/devices/af_packet/af_packet.c @@ -270,9 +270,12 @@ af_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set, sw = vnet_get_hw_sw_interface (vnm, apif->hw_if_index); apif->sw_if_index = sw->sw_if_index; - vnet_set_device_input_node (apif->hw_if_index, af_packet_input_node.index); - vnet_device_input_assign_thread (apif->hw_if_index, 0, /* queue */ + vnet_set_device_input_node (vnm, apif->hw_if_index, + af_packet_input_node.index); + vnet_device_input_assign_thread (vnm, apif->hw_if_index, 0, /* queue */ ~0 /* any cpu */ ); + vnet_device_input_set_mode (vnm, apif->hw_if_index, 0, + VNET_DEVICE_INPUT_MODE_INTERRUPT); vnet_hw_interface_set_flags (vnm, apif->hw_if_index, VNET_HW_INTERFACE_FLAG_LINK_UP); diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c index 76980102..d3af41b5 100644 --- a/src/vnet/devices/af_packet/node.c +++ b/src/vnet/devices/af_packet/node.c @@ -251,7 +251,7 @@ af_packet_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vnet_device_input_runtime_t *rt = (void *) node->runtime_data; vnet_device_and_queue_t *dq; - vec_foreach (dq, rt->devices_and_queues) + foreach_device_and_queue (dq, rt->devices_and_queues) { af_packet_if_t *apif; apif = vec_elt_at_index (apm->interfaces, dq->dev_instance); diff --git a/src/vnet/devices/devices.c b/src/vnet/devices/devices.c index 5e5e812c..c8a95087 100644 --- a/src/vnet/devices/devices.c +++ b/src/vnet/devices/devices.c @@ -102,11 +102,26 @@ vnet_device_queue_sort (void *a1, void *a2) return 0; } +static void +vnet_device_queue_update (vnet_main_t * vnm, vnet_device_input_runtime_t * rt) +{ + vnet_device_and_queue_t *dq; + vnet_hw_interface_t *hw; + + vec_sort_with_function (rt->devices_and_queues, vnet_device_queue_sort); + + vec_foreach (dq, rt->devices_and_queues) + { + hw = vnet_get_hw_interface (vnm, dq->hw_if_index); + vec_validate (hw->dq_runtime_index_by_queue, dq->queue_id); + hw->dq_runtime_index_by_queue[dq->queue_id] = dq - rt->devices_and_queues; + } +} + void -vnet_device_input_assign_thread (u32 hw_if_index, +vnet_device_input_assign_thread (vnet_main_t * vnm, u32 hw_if_index, u16 queue_id, uword thread_index) { - vnet_main_t *vnm = vnet_get_main (); vnet_device_main_t *vdm = &vnet_device_main; vlib_main_t *vm; vnet_device_input_runtime_t *rt; @@ -135,16 +150,17 @@ vnet_device_input_assign_thread (u32 hw_if_index, dq->dev_instance = hw->dev_instance; dq->queue_id = queue_id; - vec_sort_with_function (rt->devices_and_queues, vnet_device_queue_sort); + vnet_device_queue_update (vnm, rt); vec_validate (hw->input_node_thread_index_by_queue, queue_id); hw->input_node_thread_index_by_queue[queue_id] = thread_index; + vlib_node_set_state (vm, hw->input_node_index, rt->enabled_node_state); } -static int -vnet_device_input_unassign_thread (u32 hw_if_index, u16 queue_id, - uword thread_index) +int +vnet_device_input_unassign_thread (vnet_main_t * vnm, u32 hw_if_index, + u16 queue_id, uword thread_index) { - vnet_main_t *vnm = vnet_get_main (); + vlib_main_t *vm; vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); vnet_device_input_runtime_t *rt; vnet_device_and_queue_t *dq; @@ -161,9 +177,9 @@ vnet_device_input_unassign_thread (u32 hw_if_index, u16 queue_id, if (old_thread_index == thread_index) return 0; - rt = - vlib_node_get_runtime_data (vlib_mains[old_thread_index], - hw->input_node_index); + vm = vlib_mains[old_thread_index]; + + rt = vlib_node_get_runtime_data (vm, hw->input_node_index); vec_foreach (dq, rt->devices_and_queues) if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id) @@ -175,11 +191,89 @@ vnet_device_input_unassign_thread (u32 hw_if_index, u16 queue_id, return VNET_API_ERROR_INVALID_INTERFACE; deleted: - vec_sort_with_function (rt->devices_and_queues, vnet_device_queue_sort); + + vnet_device_queue_update (vnm, rt); + + if (vec_len (rt->devices_and_queues) == 0) + vlib_node_set_state (vm, hw->input_node_index, VLIB_NODE_STATE_DISABLED); + + return 0; +} + + +int +vnet_device_input_set_mode (vnet_main_t * vnm, u32 hw_if_index, u16 queue_id, + vnet_device_input_mode_t mode) +{ + vlib_main_t *vm; + uword thread_index; + vnet_device_and_queue_t *dq; + vlib_node_state_t enabled_node_state; + ASSERT (mode < VNET_DEVICE_INPUT_N_MODES); + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); + vnet_device_input_runtime_t *rt; + int is_polling = 0; + + if (hw->input_node_thread_index_by_queue == 0) + return VNET_API_ERROR_INVALID_INTERFACE; + + thread_index = hw->input_node_thread_index_by_queue[queue_id]; + vm = vlib_mains[thread_index]; + + rt = vlib_node_get_runtime_data (vm, hw->input_node_index); + + vec_foreach (dq, rt->devices_and_queues) + { + if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id) + dq->mode = mode; + if (dq->mode == VNET_DEVICE_INPUT_MODE_POLLING) + is_polling = 1; + } + + if (is_polling) + enabled_node_state = VLIB_NODE_STATE_POLLING; + else + enabled_node_state = VLIB_NODE_STATE_INTERRUPT; + + if (rt->enabled_node_state != enabled_node_state) + { + rt->enabled_node_state = enabled_node_state; + if (vlib_node_get_state (vm, hw->input_node_index) != + VLIB_NODE_STATE_DISABLED) + vlib_node_set_state (vm, hw->input_node_index, enabled_node_state); + } return 0; } +int +vnet_device_input_get_mode (vnet_main_t * vnm, u32 hw_if_index, u16 queue_id, + vnet_device_input_mode_t * mode) +{ + vlib_main_t *vm; + uword thread_index; + vnet_device_and_queue_t *dq; + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); + vnet_device_input_runtime_t *rt; + + if (hw->input_node_thread_index_by_queue == 0) + return VNET_API_ERROR_INVALID_INTERFACE; + + thread_index = hw->input_node_thread_index_by_queue[queue_id]; + vm = vlib_mains[thread_index]; + + rt = vlib_node_get_runtime_data (vm, hw->input_node_index); + + vec_foreach (dq, rt->devices_and_queues) + if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id) + { + *mode = dq->mode; + return 0; + } + + return VNET_API_ERROR_INVALID_INTERFACE; +} + static clib_error_t * show_device_placement_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) @@ -203,9 +297,11 @@ show_device_placement_fn (vlib_main_t * vm, unformat_input_t * input, vec_foreach (dq, rt->devices_and_queues) { - s = format (s, " %U queue %u\n", + s = format (s, " %U queue %u (%s)\n", format_vnet_sw_if_index_name, vnm, dq->hw_if_index, - dq->queue_id); + dq->queue_id, + dq->mode == VNET_DEVICE_INPUT_MODE_POLLING ? + "polling" : "interrupt"); } })); if (vec_len (s) > 0) @@ -238,6 +334,7 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input, unformat_input_t _line_input, *line_input = &_line_input; vnet_main_t *vnm = vnet_get_main (); vnet_device_main_t *vdm = &vnet_device_main; + vnet_device_input_mode_t mode; u32 hw_if_index = (u32) ~ 0; u32 queue_id = (u32) 0; u32 thread_index = (u32) ~ 0; @@ -275,13 +372,19 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input, return clib_error_return (0, "please specify valid worker thread or main"); - rv = - vnet_device_input_unassign_thread (hw_if_index, queue_id, thread_index); + rv = vnet_device_input_get_mode (vnm, hw_if_index, queue_id, &mode); + + if (rv) + return clib_error_return (0, "not found"); + + rv = vnet_device_input_unassign_thread (vnm, hw_if_index, queue_id, + thread_index); if (rv) return clib_error_return (0, "not found"); - vnet_device_input_assign_thread (hw_if_index, queue_id, thread_index); + vnet_device_input_assign_thread (vnm, hw_if_index, queue_id, thread_index); + vnet_device_input_set_mode (vnm, hw_if_index, queue_id, mode); return 0; } diff --git a/src/vnet/devices/devices.h b/src/vnet/devices/devices.h index 966f8302..baf03b7c 100644 --- a/src/vnet/devices/devices.h +++ b/src/vnet/devices/devices.h @@ -55,16 +55,26 @@ typedef struct uword next_worker_thread_index; } vnet_device_main_t; +typedef enum +{ + VNET_DEVICE_INPUT_MODE_POLLING = 0, + VNET_DEVICE_INPUT_MODE_INTERRUPT, + VNET_DEVICE_INPUT_N_MODES, +} vnet_device_input_mode_t; + typedef struct { u32 hw_if_index; u32 dev_instance; u16 queue_id; + vnet_device_input_mode_t mode; + uword interrupt_pending; } vnet_device_and_queue_t; typedef struct { vnet_device_and_queue_t *devices_and_queues; + vlib_node_state_t enabled_node_state; } vnet_device_input_runtime_t; extern vnet_device_main_t vnet_device_main; @@ -72,15 +82,22 @@ extern vlib_node_registration_t device_input_node; extern const u32 device_input_next_node_advance[]; static inline void -vnet_set_device_input_node (u32 hw_if_index, u32 node_index) +vnet_set_device_input_node (vnet_main_t * vnm, u32 hw_if_index, + u32 node_index) { - vnet_main_t *vnm = vnet_get_main (); vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); hw->input_node_index = node_index; } -void vnet_device_input_assign_thread (u32 hw_if_index, u16 queue_id, - uword thread_index); +void vnet_device_input_assign_thread (vnet_main_t * vnm, u32 hw_if_index, + u16 queue_id, uword thread_index); +int vnet_device_input_unassign_thread (vnet_main_t * vnm, u32 hw_if_index, + u16 queue_id, uword thread_index); +int vnet_device_input_set_mode (vnet_main_t * vnm, u32 hw_if_index, + u16 queue_id, vnet_device_input_mode_t mode); +int vnet_device_input_get_mode (vnet_main_t * vnm, u32 hw_if_index, + u16 queue_id, + vnet_device_input_mode_t * mode); static inline u64 vnet_get_aggregate_rx_packets (void) @@ -111,18 +128,41 @@ vnet_get_device_and_queue (vlib_main_t * vm, vlib_node_runtime_t * node) return rt->devices_and_queues; } +static_always_inline uword +vnet_get_device_input_thread_index (vnet_main_t * vnm, u32 hw_if_index, + u16 queue_id) +{ + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); + ASSERT (queue_id < vec_len (hw->input_node_thread_index_by_queue)); + return hw->input_node_thread_index_by_queue[queue_id]; +} + static_always_inline void vnet_device_input_set_interrupt_pending (vnet_main_t * vnm, u32 hw_if_index, u16 queue_id) { - vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); - - ASSERT (queue_id < vec_len (hw->input_node_thread_index_by_queue)); - u32 thread_index = hw->input_node_thread_index_by_queue[queue_id]; - vlib_node_set_interrupt_pending (vlib_mains[thread_index], - hw->input_node_index); + vlib_main_t *vm; + vnet_hw_interface_t *hw; + vnet_device_input_runtime_t *rt; + vnet_device_and_queue_t *dq; + uword idx; + + hw = vnet_get_hw_interface (vnm, hw_if_index); + idx = vnet_get_device_input_thread_index (vnm, hw_if_index, queue_id); + vm = vlib_mains[idx]; + rt = vlib_node_get_runtime_data (vm, hw->input_node_index); + idx = hw->dq_runtime_index_by_queue[queue_id]; + dq = vec_elt_at_index (rt->devices_and_queues, idx); + dq->interrupt_pending = 1; + + vlib_node_set_interrupt_pending (vm, hw->input_node_index); } +#define foreach_device_and_queue(var,vec) \ + for (var = (vec); var < vec_end (vec); var++) \ + if (clib_smp_swap (&((var)->interrupt_pending), 0) || \ + var->mode == VNET_DEVICE_INPUT_MODE_POLLING) + #endif /* included_vnet_vnet_device_h */ /* diff --git a/src/vnet/interface.c b/src/vnet/interface.c index 45417b2f..24f216f6 100644 --- a/src/vnet/interface.c +++ b/src/vnet/interface.c @@ -919,6 +919,8 @@ vnet_delete_hw_interface (vnet_main_t * vnm, u32 hw_if_index) hash_unset_mem (im->hw_interface_by_name, hw->name); vec_free (hw->name); + vec_free (hw->input_node_thread_index_by_queue); + vec_free (hw->dq_runtime_index_by_queue); pool_put (im->hw_interfaces, hw); } diff --git a/src/vnet/interface.h b/src/vnet/interface.h index 08f08b10..9c223040 100644 --- a/src/vnet/interface.h +++ b/src/vnet/interface.h @@ -470,6 +470,9 @@ typedef struct vnet_hw_interface_t /* input node cpu index by queue */ u32 *input_node_thread_index_by_queue; + /* device input device_and_queue runtime index */ + uword *dq_runtime_index_by_queue; + } vnet_hw_interface_t; extern vnet_device_class_t vnet_local_interface_device_class; |