From 85ecc810ca98550a250c74f32244760e459e3f87 Mon Sep 17 00:00:00 2001 From: Jasvinder Singh Date: Thu, 21 Jul 2016 17:02:19 +0100 Subject: DPDK HQoS: Enable Hierarchical Scheduler in VPP This commit extends the vpp framework with new thread type "hqos-threads" that runs the Hierarchical Quality of Service (HQoS) scheduler associted with output interface. HQoS Scheduler prioritize the packets from different users and ensures sufficient bandwidth to pass the more important traffic. At high level, HQoS scheduler is a buffer that can temporarily store a large number of packets. In otherwords, it is a collection of large number of queues organized into hierarchy of 5 levels; the port (i.e. the physical interface) is at the root of the hierarchy followed by the subport (a set of users), the pipes (individual users), the traffic classes (each with a strict priority) and at the leaves, the queues. In each HQoS scheduler, three operations are performed; classification (setting HQoS port, subport, pipe, traffic class and queue within traffic class from packet fields), enqueue (selecting HQoS queue for the packet, and to drop the packet if the queue is full) and dequeue (schedule the packet based on its length and available credits, and handover the scheduled packet to the output interface). In vpp, the number of hqos threads will be equal to cpu cores specified in corelist-hqos-threads parameter cpu section of the vpp configuration file. One hqos thread can run HQoS for multiple output interfaces. A particular HQoS instance is initialised with default parameters required to configure hqos port, subport, pipe and queues. Some of them can be re-configured in run-time through CLI commands as well binary APIs. Following illustrates the sample startup configuration file with 4x worker threads feeding 2x hqos threads that handle each HQoS for 1x output interface. For more details on HQoS configuration please refer to DPDK Programmer's Guide. dpdk { socket-mem 16384,16384 dev 0000:02:00.0 { num-rx-queues 2 hqos } dev 0000:06:00.0 { num-rx-queues 2 hqos } num-mbufs 1000000 } cpu { main-core 0 corelist-workers 1, 2, 3, 4 corelist-hqos-threads 5, 6 } Change-Id: I635c3395a7c4ddf0a239ef77b0b0a31a6dfc4767 Signed-off-by: Cristian Dumitrescu Signed-off-by: Jasvinder Singh --- vpp/vpp-api/api.c | 144 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 142 insertions(+), 2 deletions(-) (limited to 'vpp/vpp-api/api.c') diff --git a/vpp/vpp-api/api.c b/vpp/vpp-api/api.c index 3bdc80af..6bf5fed7 100644 --- a/vpp/vpp-api/api.c +++ b/vpp/vpp-api/api.c @@ -255,6 +255,9 @@ _(SW_INTERFACE_SET_MPLS_ENABLE, sw_interface_set_mpls_enable) \ _(SW_INTERFACE_SET_VPATH, sw_interface_set_vpath) \ _(SW_INTERFACE_SET_L2_XCONNECT, sw_interface_set_l2_xconnect) \ _(SW_INTERFACE_SET_L2_BRIDGE, sw_interface_set_l2_bridge) \ +_(SW_INTERFACE_SET_DPDK_HQOS_PIPE, sw_interface_set_dpdk_hqos_pipe) \ +_(SW_INTERFACE_SET_DPDK_HQOS_SUBPORT, sw_interface_set_dpdk_hqos_subport) \ +_(SW_INTERFACE_SET_DPDK_HQOS_TCTBL, sw_interface_set_dpdk_hqos_tctbl) \ _(BRIDGE_DOMAIN_ADD_DEL, bridge_domain_add_del) \ _(BRIDGE_DOMAIN_DUMP, bridge_domain_dump) \ _(BRIDGE_DOMAIN_DETAILS, bridge_domain_details) \ @@ -1450,6 +1453,144 @@ static void REPLY_MACRO (VL_API_SW_INTERFACE_SET_L2_BRIDGE_REPLY); } +static void + vl_api_sw_interface_set_dpdk_hqos_pipe_t_handler + (vl_api_sw_interface_set_dpdk_hqos_pipe_t * mp) +{ + vl_api_sw_interface_set_dpdk_hqos_pipe_reply_t *rmp; + int rv = 0; + +#if DPDK > 0 + dpdk_main_t *dm = &dpdk_main; + dpdk_device_t *xd; + + u32 sw_if_index = ntohl (mp->sw_if_index); + u32 subport = ntohl (mp->subport); + u32 pipe = ntohl (mp->pipe); + u32 profile = ntohl (mp->profile); + vnet_hw_interface_t *hw; + + VALIDATE_SW_IF_INDEX (mp); + + /* hw_if & dpdk device */ + hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index); + + xd = vec_elt_at_index (dm->devices, hw->dev_instance); + + rv = rte_sched_pipe_config (xd->hqos_ht->hqos, subport, pipe, profile); + + BAD_SW_IF_INDEX_LABEL; +#else + clib_warning ("setting HQoS pipe parameters without DPDK not implemented"); + rv = VNET_API_ERROR_UNIMPLEMENTED; +#endif /* DPDK */ + + REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_PIPE_REPLY); +} + +static void + vl_api_sw_interface_set_dpdk_hqos_subport_t_handler + (vl_api_sw_interface_set_dpdk_hqos_subport_t * mp) +{ + vl_api_sw_interface_set_dpdk_hqos_subport_reply_t *rmp; + int rv = 0; + +#if DPDK > 0 + dpdk_main_t *dm = &dpdk_main; + dpdk_device_t *xd; + struct rte_sched_subport_params p; + + u32 sw_if_index = ntohl (mp->sw_if_index); + u32 subport = ntohl (mp->subport); + p.tb_rate = ntohl (mp->tb_rate); + p.tb_size = ntohl (mp->tb_size); + p.tc_rate[0] = ntohl (mp->tc_rate[0]); + p.tc_rate[1] = ntohl (mp->tc_rate[1]); + p.tc_rate[2] = ntohl (mp->tc_rate[2]); + p.tc_rate[3] = ntohl (mp->tc_rate[3]); + p.tc_period = ntohl (mp->tc_period); + + vnet_hw_interface_t *hw; + + VALIDATE_SW_IF_INDEX (mp); + + /* hw_if & dpdk device */ + hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index); + + xd = vec_elt_at_index (dm->devices, hw->dev_instance); + + rv = rte_sched_subport_config (xd->hqos_ht->hqos, subport, &p); + + BAD_SW_IF_INDEX_LABEL; +#else + clib_warning + ("setting HQoS subport parameters without DPDK not implemented"); + rv = VNET_API_ERROR_UNIMPLEMENTED; +#endif /* DPDK */ + + REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_SUBPORT_REPLY); +} + +static void + vl_api_sw_interface_set_dpdk_hqos_tctbl_t_handler + (vl_api_sw_interface_set_dpdk_hqos_tctbl_t * mp) +{ + vl_api_sw_interface_set_dpdk_hqos_tctbl_reply_t *rmp; + int rv = 0; + +#if DPDK > 0 + dpdk_main_t *dm = &dpdk_main; + vlib_thread_main_t *tm = vlib_get_thread_main (); + dpdk_device_t *xd; + + u32 sw_if_index = ntohl (mp->sw_if_index); + u32 entry = ntohl (mp->entry); + u32 tc = ntohl (mp->tc); + u32 queue = ntohl (mp->queue); + u32 val, i; + + vnet_hw_interface_t *hw; + + VALIDATE_SW_IF_INDEX (mp); + + /* hw_if & dpdk device */ + hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index); + + xd = vec_elt_at_index (dm->devices, hw->dev_instance); + + if (tc >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE) + { + clib_warning ("invalid traffic class !!"); + rv = VNET_API_ERROR_INVALID_VALUE; + goto done; + } + if (queue >= RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS) + { + clib_warning ("invalid queue !!"); + rv = VNET_API_ERROR_INVALID_VALUE; + goto done; + } + + /* Detect the set of worker threads */ + uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers"); + vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0]; + int worker_thread_first = tr->first_index; + int worker_thread_count = tr->count; + + val = tc * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + queue; + for (i = 0; i < worker_thread_count; i++) + xd->hqos_wt[worker_thread_first + i].hqos_tc_table[entry] = val; + + BAD_SW_IF_INDEX_LABEL; +done: +#else + clib_warning ("setting HQoS DSCP table entry without DPDK not implemented"); + rv = VNET_API_ERROR_UNIMPLEMENTED; +#endif /* DPDK */ + + REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_TCTBL_REPLY); +} + static void vl_api_bridge_domain_add_del_t_handler (vl_api_bridge_domain_add_del_t * mp) { @@ -2080,8 +2221,7 @@ static int mpls_ethernet_add_del_tunnel_2_t_handler // FIXME not an ADJ lookup_result = ip4_fib_table_lookup_lb (ip4_fib_get (outer_fib_index), (ip4_address_t *) - mp-> - next_hop_ip4_address_in_outer_vrf); + mp->next_hop_ip4_address_in_outer_vrf); adj = ip_get_adjacency (lm, lookup_result); tx_sw_if_index = adj->rewrite_header.sw_if_index; -- cgit 1.2.3-korg