/* * Copyright(c) 2016 Intel Corporation. All rights reserved. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* enumerate all vlib messages */ #define vl_typedefs /* define message structures */ #include #undef vl_typedefs /* instantiate all the print functions we know about */ #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) #define vl_printfun #include #undef vl_printfun #include "dpdk_priv.h" dpdk_main_t dpdk_main; /*** * * HQoS default configuration values * ***/ static dpdk_device_config_hqos_t hqos_params_default = { .hqos_thread_valid = 0, .swq_size = 4096, .burst_enq = 256, .burst_deq = 220, /* * Packet field to identify the subport. * * Default value: Since only one subport is defined by default (see below: * n_subports_per_port = 1), the subport ID is hardcoded to 0. */ .pktfield0_slabpos = 0, .pktfield0_slabmask = 0, /* * Packet field to identify the pipe. * * Default value: Assuming Ethernet/IPv4/UDP packets, UDP payload bits 12 .. 23 */ .pktfield1_slabpos = 40, .pktfield1_slabmask = 0x0000000FFF000000LLU, /* Packet field used as index into TC translation table to identify the traffic * class and queue. * * Default value: Assuming Ethernet/IPv4 packets, IPv4 DSCP field */ .pktfield2_slabpos = 8, .pktfield2_slabmask = 0x00000000000000FCLLU, .tc_table = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, }, /* port */ .port = { .name = NULL, /* Set at init */ .socket = 0, /* Set at init */ .rate = 1250000000, /* Assuming 10GbE port */ .mtu = 14 + 1500, /* Assuming Ethernet/IPv4 pkt (Ethernet FCS not included) */ .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT, .n_subports_per_port = 1, .n_pipes_per_subport = 4096, .qsize = {64, 64, 64, 64}, .pipe_profiles = NULL, /* Set at config */ .n_pipe_profiles = 1, #ifdef RTE_SCHED_RED .red_params = { /* Traffic Class 0 Colors Green / Yellow / Red */ [0][0] = {.min_th = 48,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9}, [0][1] = {.min_th = 40,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9}, [0][2] = {.min_th = 32,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9}, /* Traffic Class 1 - Colors Green / Yellow / Red */ [1][0] = {.min_th = 48,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9}, [1][1] = {.min_th = 40,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9}, [1][2] = {.min_th = 32,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9}, /* Traffic Class 2 - Colors Green / Yellow / Red */ [2][0] = {.min_th = 48,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9}, [2][1] = {.min_th = 40,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9}, [2][2] = {.min_th = 32,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9}, /* Traffic Class 3 - Colors Green / Yellow / Red */ [3][0] = {.min_th = 48,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9}, [3][1] = {.min_th = 40,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9}, [3][2] = {.min_th = 32,.max_th = 64,.maxp_inv = 10,.wq_log2 = 9} }, #endif /* RTE_SCHED_RED */ }, }; static struct rte_sched_subport_params hqos_subport_params_default = { .tb_rate = 1250000000, /* 10GbE line rate (measured in bytes/second) */ .tb_size = 1000000, .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000}, .tc_period = 10, }; static struct rte_sched_pipe_params hqos_pipe_params_default = { .tb_rate = 305175, /* 10GbE line rate divided by 4K pipes */ .tb_size = 1000000, .tc_rate = {305175, 305175, 305175, 305175}, .tc_period = 40, #ifdef RTE_SCHED_SUBPORT_TC_OV .tc_ov_weight = 1, #endif .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, }; /*** * * HQoS configuration * ***/ int dpdk_hqos_validate_mask (u64 mask, u32 n) { int count = __builtin_popcountll (mask); int pos_lead = sizeof (u64) * 8 - __builtin_clzll (mask); int pos_trail = __builtin_ctzll (mask); int count_expected = __builtin_popcount (n - 1); /* Handle the exceptions */ if (n == 0) return -1; /* Error */ if ((mask == 0) && (n == 1)) return 0; /* OK */ if (((mask == 0) && (n != 1)) || ((mask != 0) && (n == 1))) return -2; /* Error */ /* Check that mask is contiguous */ if ((pos_lead - pos_trail) != count) return -3; /* Error */ /* Check that mask contains the expected number of bits set */ if (count != count_expected) return -4; /* Error */ return 0; /* OK */ } void dpdk_device_config_hqos_pipe_profile_default (dpdk_device_config_hqos_t * hqos, u32 pipe_profile_id) { memcpy (&hqos->pipe[pipe_profile_id], &hqos_pipe_params_default, sizeof (hqos_pipe_params_default)); } void dpdk_device_config_hqos_default (dpdk_device_config_hqos_t * hqos) { struct rte_sched_subport_params *subport_params; struct rte_sched_pipe_params *pipe_params; u32 *pipe_map; u32 i; memcpy (hqos, &hqos_params_default, sizeof (hqos_params_default)); /* pipe */ vec_add2 (hqos->pipe, pipe_params, hqos->port.n_pipe_profiles); for (i = 0; i < vec_len (hqos->pipe); i++) memcpy (&pipe_params[i], &hqos_pipe_params_default, sizeof (hqos_pipe_params_default)); hqos->port.pipe_profiles = hqos->pipe; /* subport */ vec_add2 (hqos->subport, subport_params, hqos->port.n_subports_per_port); for (i = 0; i < vec_len (hqos->subport); i++) memcpy (&subport_params[i], &hqos_subport_params_default, sizeof (hqos_subport_params_default)); /* pipe profile */ vec_add2 (hqos->pipe_map, pipe_map, hqos->port.n_subports_per_port * hqos->port.n_pipes_per_subport); for (i = 0; i < vec_len (hqos->pipe_map); i++) pipe_map[i] = 0; } /*** * * HQoS init * ***/ clib_error_t * dpdk_port_setup_hqos (dpdk_device_t * xd, dpdk_device_config_hqos_t * hqos) { vlib_thread_main_t *tm = vlib_get_thread_main (); char name[32]; u32 subport_id, i; int rv; /* Detect the set of worker threads */ int worker_thread_first = 0; int worker_thread_count = 0; uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers"); vlib_thread_registration_t *tr = p ? (vlib_thread_registration_t *) p[0] : 0; if (tr && tr->count > 0) { worker_thread_first = tr->first_index; worker_thread_count = tr->count; } /* Allocate the per-thread device data array */ vec_validate_aligned (xd->hqos_wt, tm->n_vlib_mains - 1, CLIB_CACHE_LINE_BYTES); memset (xd->hqos_wt, 0, tm->n_vlib_mains * sizeof (xd->hqos_wt[0])); vec_validate_aligned (xd->hqos_ht, 0, CLIB_CACHE_LINE_BYTES); memset (xd->hqos_ht, 0, sizeof (xd->hqos_ht[0])); /* Allocate space for one SWQ per worker thread in the I/O TX thread data structure */ vec_validate (xd->hqos_ht->swq, worker_thread_count); /* SWQ */ for (i = 0; i < worker_thread_count + 1; i++) { u32 swq_flags = RING_F_SP_ENQ | RING_F_SC_DEQ; snprintf (name, sizeof (name), "SWQ-worker%u-to-device%u", i, xd->device_index); xd->hqos_ht->swq[i] = rte_ring_create (name, hqos->swq_size, xd->cpu_socket, swq_flags); if (xd->hqos_ht->swq[i] == NULL) return clib_error_return (0, "SWQ-worker%u-to-device%u: rte_ring_create err", i, xd->device_index); } /* * HQoS */ /* HQoS port */ snprintf (name, sizeof (name), "HQoS%u", xd->device_index); hqos->port.name = strdup (name); if (hqos->port.name == NULL) return clib_error_return (0, "HQoS%u: strdup err", xd->device_index); hqos->port.socket = rte_eth_dev_socket_id (xd->device_index); if (hqos->port.socket == SOCKET_ID_ANY) hqos->port.socket = 0; xd->hqos_ht->hqos = rte_sched_port_config (&hqos->port); if (xd->hqos_ht->hqos == NULL) return clib_error_return (0, "HQoS%u: rte_sched_port_config err", xd->device_index); /* HQoS subport */ for (subport_id = 0; subport_id < hqos->port.n_subports_per_port; subport_id++) { u32 pipe_id; rv = rte_sched_subport_config (xd->hqos_ht->hqos, subport_id, &hqos->subport[subport_id]); if (rv) return clib_error_return (0, "HQoS%u subport %u: rte_sched_subport_config err (%d)", xd->device_index, subport_id, rv); /* HQoS pipe */ for (pipe_id = 0; pipe_id < hqos->port.n_pipes_per_subport; pipe_id++) { u32 pos = subport_id * hqos->port.n_pipes_per_subport + pipe_id; u32 profile_id = hqos->pipe_map[pos]; rv = rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id, profile_id); if (rv) return clib_error_return (0, "HQoS%u subport %u pipe %u: rte_sched_pipe_config err (%d)", xd->device_index, subport_id, pipe_id, rv); } } /* Set up per-thread device data for the I/O TX thread */ xd->hqos_ht->hqos_burst_enq = hqos->burst_enq; xd->hqos_ht->hqos_burst_deq = hqos->burst_deq; vec_validate (xd->hqos_ht->pkts_enq, 2 * hqos->burst_enq - 1); vec_validate (xd->hqos_ht->pkts_deq, hqos->burst_deq - 1); xd->hqos_ht->pkts_enq_len = 0; xd->hqos_ht->swq_pos = 0; xd->hqos_ht->flush_count = 0; /* Set up per-thread device data for each worker thread */ for (i = 0; i < worker_thread_count + 1; i++) { u32 tid; if (i) tid = worker_thread_first + (i - 1); else tid = i; xd->hqos_wt[tid].swq = xd->hqos_ht->swq[i]; xd->hqos_wt[tid].hqos_field0_slabpos = hqos->pktfield0_slabpos; xd->hqos_wt[tid].hqos_field0_slabmask = hqos->pktfield0_slabmask; xd->hqos_wt[tid].hqos_field0_slabshr = __builtin_ctzll (hqos->pktfield0_slabmask); xd->hqos_wt[tid].hqos_field1_slabpos = hqos->pktfield1_slabpos; xd->hqos_wt[tid].hqos_field1_slabmask = hqos->pktfield1_slabmask; xd->hqos_wt[tid].hqos_field1_slabshr = __builtin_ctzll (hqos->pktfield1_slabmask); xd->hqos_wt[tid].hqos_field2_slabpos = hqos->pktfield2_slabpos; xd->hqos_wt[tid].hqos_field2_slabmask = hqos->pktfield2_slabmask; xd->hqos_wt[tid].hqos_field2_slabshr = __builtin_ctzll (hqos->pktfield2_slabmask); memcpy (xd->hqos_wt[tid].hqos_tc_table, hqos->tc_table, sizeof (hqos->tc_table)); } return 0; } /*** * * HQoS run-time * ***/ /* * dpdk_hqos_thread - Contains the main loop of an HQoS thread. * * w * Information for the current thread */ static_always_inline void dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm) { dpdk_main_t *dm = &dpdk_main; u32 cpu_index = vm->cpu_index; u32 dev_pos; dev_pos = 0; while (1) { vlib_worker_thread_barrier_check (); u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]); if (dev_pos >= n_devs) dev_pos = 0; dpdk_device_and_queue_t *dq = vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos); dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device); dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht; u32 device_index = xd->device_index; u16 queue_id = dq->queue_id; struct rte_mbuf **pkts_enq = hqos->pkts_enq; u32 pkts_enq_len = hqos->pkts_enq_len; u32 swq_pos = hqos->swq_pos; u32 n_swq = vec_len (hqos->swq), i; u32 flush_count = hqos->flush_count; for (i = 0; i < n_swq; i++) { /* Get current SWQ for this device */ struct rte_ring *swq = hqos->swq[swq_pos]; /* Read SWQ burst to packet buffer of this device */ pkts_enq_len += rte_ring_sc_dequeue_burst (swq, (void **) &pkts_enq[pkts_enq_len], hqos->hqos_burst_enq); /* Get next SWQ for this device */ swq_pos++; if (swq_pos >= n_swq) swq_pos = 0; hqos->swq_pos = swq_pos; /* HWQ TX enqueue when burst available */ if (pkts_enq_len >= hqos->hqos_burst_enq) { u32 n_pkts = rte_eth_tx_burst (device_index, (uint16_t) queue_id, pkts_enq, (uint16_t) pkts_enq_len); for (; n_pkts < pkts_enq_len; n_pkts++) rte_pktmbuf_free (pkts_enq[n_pkts]); pkts_enq_len = 0; flush_count = 0; break; } } if (pkts_enq_len) { flush_count++; if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD)) { rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len); pkts_enq_len = 0; flush_count = 0; } } hqos->pkts_enq_len = pkts_enq_len; hqos->flush_count = flush_count; /* Advance to next device */ dev_pos++; } } static_always_inline void dpdk_hqos_thread_internal (vlib_main_t * vm) { dpdk_main_t *dm = &dpdk_main; u32 cpu_index = vm->cpu_index; u32 dev_pos; dev_pos = 0; while (1) { vlib_worker_thread_barrier_check (); u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]); if (PREDICT_FALSE (n_devs == 0)) { dev_pos = 0; continue; } if (dev_pos >= n_devs) dev_pos = 0; dpdk_device_and_queue_t *dq = vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos); dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device); dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht; u32 device_index = xd->device_index; u16 queue_id = dq->queue_id; struct rte_mbuf **pkts_enq = hqos->pkts_enq; struct rte_mbuf **pkts_deq = hqos->pkts_deq; u32 pkts_enq_len = hqos->pkts_enq_len; u32 swq_pos = hqos->swq_pos; u32 n_swq = vec_len (hqos->swq), i; u32 flush_count = hqos->flush_count; /* * SWQ dequeue and HQoS enqueue for current device */ for (i = 0; i < n_swq; i++) { /* Get current SWQ for this device */ struct rte_ring *swq = hqos->swq[swq_pos]; /* Read SWQ burst to packet buffer of this device */ pkts_enq_len += rte_ring_sc_dequeue_burst (swq, (void **) &pkts_enq[pkts_enq_len], hqos->hqos_burst_enq); /* Get next SWQ for this device */ swq_pos++; if (swq_pos >= n_swq) swq_pos = 0; hqos->swq_pos = swq_pos; /* HQoS enqueue when burst available */ if (pkts_enq_len >= hqos->hqos_burst_enq) { rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len); pkts_enq_len = 0; flush_count = 0; break; } } if (pkts_enq_len) { flush_count++; if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD)) { rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len); pkts_enq_len = 0; flush_count = 0; } } hqos->pkts_enq_len = pkts_enq_len; hqos->flush_count = flush_count; /* * HQoS dequeue and HWQ TX enqueue for current device */ { u32 pkts_deq_len, n_pkts; pkts_deq_len = rte_sched_port_dequeue (hqos->hqos, pkts_deq, hqos->hqos_burst_deq); for (n_pkts = 0; n_pkts < pkts_deq_len;) n_pkts += rte_eth_tx_burst (device_index, (uint16_t) queue_id, &pkts_deq[n_pkts], (uint16_t) (pkts_deq_len - n_pkts)); } /* Advance to next device */ dev_pos++; } } void dpdk_hqos_thread (vlib_worker_thread_t * w) { vlib_main_t *vm; vlib_thread_main_t *tm = vlib_get_thread_main (); dpdk_main_t *dm = &dpdk_main; vm = vlib_get_main (); ASSERT (vm->cpu_index == os_get_cpu_number ()); clib_time_init (&vm->clib_time); clib_mem_set_heap (w->thread_mheap); /* Wait until the dpdk init sequence is complete */ while (tm->worker_thread_release == 0) vlib_worker_thread_barrier_check (); if (vec_len (dm->devices_by_hqos_cpu[vm->cpu_index]) == 0) return clib_error ("current I/O TX thread does not have any devices assigned to it"); if (DPDK_HQOS_DBG_BYPASS) dpdk_hqos_thread_internal_hqos_dbg_bypass (vm); else dpdk_hqos_thread_internal (vm); } void dpdk_hqos_thread_fn (void *arg) { vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; vlib_worker_thread_init (w); dpdk_hqos_thread (w); } /* *INDENT-OFF* */ VLIB_REGISTER_THREAD (hqos_thread_reg, static) = { .name = "hqos-threads", .short_name = "hqos-threads", .function = dpdk_hqos_thread_fn, }; /* *INDENT-ON* */ /* * HQoS run-time code to be called by the worker threads */ #define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr) \ ({ \ u64 slab = *((u64 *) &byte_array[slab_pos]); \ u64 val = (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr; \ val; \ }) #define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color) \ ((((u64) (queue)) & 0x3) | \ ((((u64) (traffic_class)) & 0x3) << 2) | \ ((((u64) (color)) & 0x3) << 4) | \ ((((u64) (subport)) & 0xFFFF) << 16) | \ ((((u64) (pipe)) & 0xFFFFFFFF) << 32)) void dpdk_hqos_metadata_set (dpdk_device_hqos_per_worker_thread_t * hqos, struct rte_mbuf **pkts, u32 n_pkts) { u32 i; for (i = 0; i < (n_pkts & (~0x3)); i += 4) { struct rte_mbuf *pkt0 = pkts[i]; struct rte_mbuf *pkt1 = pkts[i + 1]; struct rte_mbuf *pkt2 = pkts[i + 2]; struct rte_mbuf *pkt3 = pkts[i + 3]; u8 *pkt0_data = rte_pktmbuf_mtod (pkt0, u8 *); u8 *pkt1_data = rte_pktmbuf_mtod (pkt1, u8 *); u8 *pkt2_data = rte_pktmbuf_mtod (pkt2, u8 *); u8 *pkt3_data = rte_pktmbuf_mtod (pkt3, u8 *); u64 pkt0_subport = BITFIELD (pkt0_data, hqos->hqos_field0_slabpos, hqos->hqos_field0_slabmask, hqos->hqos_field0_slabshr); u64 pkt0_pipe = BITFIELD (pkt0_data, hqos->hqos_field1_slabpos, hqos->hqos_field1_slabmask, hqos->hqos_field1_slabshr); u64 pkt0_dscp = BITFIELD (pkt0_data, hqos->hqos_field2_slabpos, hqos->hqos_field2_slabmask, hqos->hqos_field2_slabshr); u32 pkt0_tc = hqos->hqos_tc_table[pkt0_dscp & 0x3F] >> 2; u32 pkt0_tc_q = hqos->hqos_tc_table[pkt0_dscp & 0x3F] & 0x3; u64 pkt1_subport = BITFIELD (pkt1_data, hqos->hqos_field0_slabpos, hqos->hqos_field0_slabmask, hqos->hqos_field0_slabshr); u64 pkt1_pipe = BITFIELD (pkt1_data, hqos->hqos_field1_slabpos, hqos->hqos_field1_slabmask, hqos->hqos_field1_slabshr); u64 pkt1_dscp = BITFIELD (pkt1_data, hqos->hqos_field2_slabpos, hqos->hqos_field2_slabmask, hqos->hqos_field2_slabshr); u32 pkt1_tc = hqos->hqos_tc_table[pkt1_dscp & 0x3F] >> 2; u32 pkt1_tc_q = hqos->hqos_tc_table[pkt1_dscp & 0x3F] & 0x3; u64 pkt2_subport = BITFIELD (pkt2_data, hqos->hqos_field0_slabpos, hqos->hqos_field0_slabmask, hqos->hqos_field0_slabshr); u64 pkt2_pipe = BITFIELD (pkt2_data, hqos->hqos_field1_slabpos, hqos->hqos_field1_slabmask, hqos->hqos_field1_slabshr); u64 pkt2_dscp = BITFIELD (pkt2_data, hqos->hqos_field2_slabpos, hqos->hqos_field2_slabmask, hqos->hqos_field2_slabshr); u32 pkt2_tc = hqos->hqos_tc_table[pkt2_dscp & 0x3F] >> 2; u32 pkt2_tc_q = hqos->hqos_tc_table[pkt2_dscp & 0x3F] & 0x3; u64 pkt3_subport = BITFIELD (pkt3_data, hqos->hqos_field0_slabpos, hqos->hqos_field0_slabmask, hqos->hqos_field0_slabshr); u64 pkt3_pipe = BITFIELD (pkt3_data, hqos->hqos_field1_slabpos, hqos->hqos_field1_slabmask, hqos->hqos_field1_slabshr); u64 pkt3_dscp = BITFIELD (pkt3_data, hqos->hqos_field2_slabpos, hqos->hqos_field2_slabmask, hqos->hqos_field2_slabshr); u32 pkt3_tc = hqos->hqos_tc_table[pkt3_dscp & 0x3F] >> 2; u32 pkt3_tc_q = hqos->hqos_tc_table[pkt3_dscp & 0x3F] & 0x3; u64 pkt0_sched = RTE_SCHED_PORT_HIERARCHY (pkt0_subport, pkt0_pipe, pkt0_tc, pkt0_tc_q, 0); u64 pkt1_sched = RTE_SCHED_PORT_HIERARCHY (pkt1_subport, pkt1_pipe, pkt1_tc, pkt1_tc_q, 0); u64 pkt2_sched = RTE_SCHED_PORT_HIERARCHY (pkt2_subport, pkt2_pipe, pkt2_tc, pkt2_tc_q, 0); u64 pkt3_sched = RTE_SCHED_PORT_HIERARCHY (pkt3_subport, pkt3_pipe, pkt3_tc, pkt3_tc_q, 0); pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF; pkt0->hash.sched.hi = pkt0_sched >> 32; pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF; pkt1->hash.sched.hi = pkt1_sched >> 32; pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF; pkt2->hash.sched.hi = pkt2_sched >> 32; pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF; pkt3->hash.sched.hi = pkt3_sched >> 32; } for (; i < n_pkts; i++) { struct rte_mbuf *pkt = pkts[i]; u8 *pkt_data = rte_pktmbuf_mtod (pkt, u8 *); u64 pkt_subport = BITFIELD (pkt_data, hqos->hqos_field0_slabpos, hqos->hqos_field0_slabmask, hqos->hqos_field0_slabshr); u64 pkt_pipe = BITFIELD (pkt_data, hqos->hqos_field1_slabpos, hqos->hqos_field1_slabmask, hqos->hqos_field1_slabshr); u64 pkt_dscp = BITFIELD (pkt_data, hqos->hqos_field2_slabpos, hqos->hqos_field2_slabmask, hqos->hqos_field2_slabshr); u32 pkt_tc = hqos->hqos_tc_table[pkt_dscp & 0x3F] >> 2; u32 pkt_tc_q = hqos->hqos_tc_table[pkt_dscp & 0x3F] & 0x3; u64 pkt_sched = RTE_SCHED_PORT_HIERARCHY (pkt_subport, pkt_pipe, pkt_tc, pkt_tc_q, 0); pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF; pkt->hash.sched.hi = pkt_sched >> 32; } } /* * fd.io coding-style-patch-verification: ON * * Local Variables: * eval: (c-set-style "gnu") * End: */