diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/plugins/dev_iavf/adminq.c | 68 | ||||
| -rw-r--r-- | src/plugins/dev_iavf/iavf.c | 22 | ||||
| -rw-r--r-- | src/plugins/dev_iavf/iavf.h | 5 | ||||
| -rw-r--r-- | src/plugins/dev_iavf/iavf_regs.h | 17 | ||||
| -rw-r--r-- | src/plugins/dev_iavf/port.c | 151 | ||||
| -rw-r--r-- | src/plugins/dev_iavf/virtchnl.h | 34 | ||||
| -rw-r--r-- | src/vppinfra/bitops.h | 30 |
7 files changed, 241 insertions, 86 deletions
diff --git a/src/plugins/dev_iavf/adminq.c b/src/plugins/dev_iavf/adminq.c index 982a9a9d368..c12dc8aa2f6 100644 --- a/src/plugins/dev_iavf/adminq.c +++ b/src/plugins/dev_iavf/adminq.c @@ -8,6 +8,7 @@ #include <vnet/dev/pci.h> #include <vnet/dev/counters.h> #include <dev_iavf/iavf.h> +#include <dev_iavf/iavf_regs.h> #include <dev_iavf/virtchnl.h> #include <vnet/ethernet/ethernet.h> @@ -34,6 +35,36 @@ struct iavf_adminq_dma_mem } arq_bufs[IIAVF_AQ_ARQ_LEN]; }; +static const iavf_dyn_ctl dyn_ctl0_disable = { + .itr_indx = 3, +}; + +static const iavf_dyn_ctl dyn_ctl0_enable = { + .intena = 1, + .clearpba = 1, + .itr_indx = 3, +}; + +static const iavf_vfint_icr0_ena1 icr0_ena1_aq_enable = { + .adminq = 1, +}; + +static inline void +iavf_irq_0_disable (iavf_device_t *ad) +{ + iavf_reg_write (ad, IAVF_VFINT_ICR0_ENA1, 0); + iavf_reg_write (ad, IAVF_VFINT_DYN_CTL0, dyn_ctl0_disable.as_u32); + iavf_reg_flush (ad); +} + +static inline void +iavf_irq_0_enable (iavf_device_t *ad) +{ + iavf_reg_write (ad, IAVF_VFINT_ICR0_ENA1, icr0_ena1_aq_enable.as_u32); + iavf_reg_write (ad, IAVF_VFINT_DYN_CTL0, dyn_ctl0_enable.as_u32); + iavf_reg_flush (ad); +} + static_always_inline int iavf_aq_desc_is_done (iavf_aq_desc_t *d) { @@ -242,39 +273,12 @@ iavf_aq_poll (vlib_main_t *vm, vnet_dev_t *dev) } } -static inline void -iavf_irq_0_set_state (iavf_device_t *ad, int enable) -{ - u32 dyn_ctl0 = 0, icr0_ena = 0; - - dyn_ctl0 |= (3 << 3); /* 11b = No ITR update */ - - iavf_reg_write (ad, AVFINT_ICR0_ENA1, icr0_ena); - iavf_reg_write (ad, AVFINT_DYN_CTL0, dyn_ctl0); - iavf_reg_flush (ad); - - if (!enable) - return; - - dyn_ctl0 = 0; - icr0_ena = 0; - - icr0_ena |= (1 << 30); /* [30] Admin Queue Enable */ - - dyn_ctl0 |= (1 << 0); /* [0] Interrupt Enable */ - dyn_ctl0 |= (1 << 1); /* [1] Clear PBA */ - dyn_ctl0 |= (2 << 3); /* [4:3] ITR Index, 11b = No ITR update */ - dyn_ctl0 |= ((IAVF_ITR_INT / 2) << 5); /* [16:5] ITR Interval in 2us steps */ - - iavf_reg_write (ad, AVFINT_ICR0_ENA1, icr0_ena); - iavf_reg_write (ad, AVFINT_DYN_CTL0, dyn_ctl0); - iavf_reg_flush (ad); -} - static void iavf_adminq_msix_handler (vlib_main_t *vm, vnet_dev_t *dev, u16 line) { - log_debug (dev, "MSI-X interrupt 0 received"); + iavf_device_t *ad = vnet_dev_get_data (dev); + iavf_reg_write (ad, IAVF_VFINT_DYN_CTL0, dyn_ctl0_enable.as_u32); + log_debug (dev, "MSI-X interrupt %u received", line); vnet_dev_process_call_op_no_wait (vm, dev, iavf_aq_poll); } @@ -335,7 +339,7 @@ iavf_aq_poll_on (vlib_main_t *vm, vnet_dev_t *dev) else vnet_dev_pci_intx_add_handler (vm, dev, iavf_adminq_intx_handler); - iavf_irq_0_set_state (ad, 1); + iavf_irq_0_enable (ad); } void @@ -343,7 +347,7 @@ iavf_aq_poll_off (vlib_main_t *vm, vnet_dev_t *dev) { iavf_device_t *ad = vnet_dev_get_data (dev); - iavf_irq_0_set_state (ad, 0); + iavf_irq_0_disable (ad); vnet_dev_poll_dev_remove (vm, dev, iavf_aq_poll); diff --git a/src/plugins/dev_iavf/iavf.c b/src/plugins/dev_iavf/iavf.c index a8b108c92b7..f45bea0e669 100644 --- a/src/plugins/dev_iavf/iavf.c +++ b/src/plugins/dev_iavf/iavf.c @@ -98,7 +98,7 @@ iavf_reset (vlib_main_t *vm, vnet_dev_t *dev) return VNET_DEV_ERR_TIMEOUT; vlib_process_suspend (vm, 0.02); } - while ((iavf_reg_read (ad, VFGEN_RSTAT) & 3) != 2); + while ((iavf_reg_read (ad, IAVF_VFGEN_RSTAT) & 3) != 2); iavf_aq_init (vm, dev); iavf_aq_poll_on (vm, dev); @@ -118,6 +118,7 @@ iavf_init (vlib_main_t *vm, vnet_dev_t *dev) iavf_device_t *ad = vnet_dev_get_data (dev); virtchnl_version_info_t ver; virtchnl_vf_resource_t res; + u32 n_threads = vlib_get_n_threads (); vnet_dev_rv_t rv; log_debug (dev, "init"); @@ -209,13 +210,20 @@ iavf_init (vlib_main_t *vm, vnet_dev_t *dev) log_info (dev, "MAC address is %U", format_ethernet_address, res.vsi_res[0].default_mac_addr); - if (vlib_get_n_threads () <= vnet_dev_get_pci_n_msix_interrupts (dev) - 1) - port_add_args.port.attr.caps.interrupt_mode = 1; + if (n_threads <= vnet_dev_get_pci_n_msix_interrupts (dev) - 1) + { + port_add_args.port.attr.caps.interrupt_mode = 1; + iavf_port.n_rx_vectors = n_threads; + } else - log_notice (dev, - "number of threads (%u) bigger than number of interrupt lines " - "(%u), interrupt mode disabled", - vlib_get_n_threads (), res.max_vectors); + { + log_notice ( + dev, + "number of threads (%u) bigger than number of interrupt lines " + "(%u), interrupt mode disabled", + vlib_get_n_threads (), res.max_vectors); + iavf_port.n_rx_vectors = 1; + } if (res.vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { diff --git a/src/plugins/dev_iavf/iavf.h b/src/plugins/dev_iavf/iavf.h index 958a8dcf808..47e86e14f43 100644 --- a/src/plugins/dev_iavf/iavf.h +++ b/src/plugins/dev_iavf/iavf.h @@ -32,7 +32,6 @@ typedef struct u16 atq_next_slot; u16 arq_next_slot; virtchnl_pf_event_t *events; - } iavf_device_t; typedef struct @@ -47,12 +46,14 @@ typedef struct u8 admin_up : 1; u8 flow_offload : 1; iavf_flow_lookup_entry_t *flow_lookup_entries; + u64 intr_mode_per_rxq_bitmap; u32 vf_cap_flags; u16 vsi_id; u16 rss_key_size; u16 rss_lut_size; u16 num_qp; u16 max_vectors; + u16 n_rx_vectors; } iavf_port_t; typedef struct @@ -151,7 +152,7 @@ iavf_reg_read (iavf_device_t *ad, u32 addr) static inline void iavf_reg_flush (iavf_device_t *ad) { - iavf_reg_read (ad, VFGEN_RSTAT); + iavf_reg_read (ad, IAVF_VFGEN_RSTAT); asm volatile("" ::: "memory"); } diff --git a/src/plugins/dev_iavf/iavf_regs.h b/src/plugins/dev_iavf/iavf_regs.h index 3c270d77007..be3070b05e5 100644 --- a/src/plugins/dev_iavf/iavf_regs.h +++ b/src/plugins/dev_iavf/iavf_regs.h @@ -344,8 +344,21 @@ typedef union u32 intena_msk : 1; }; u32 as_u32; -} iavf_dyn_ctln; +} iavf_dyn_ctl; -STATIC_ASSERT_SIZEOF (iavf_dyn_ctln, 4); +STATIC_ASSERT_SIZEOF (iavf_dyn_ctl, 4); + +typedef union +{ + struct + { + u32 _reserved0 : 30; + u32 adminq : 1; + u32 _reserved31 : 1; + }; + u32 as_u32; +} iavf_vfint_icr0_ena1; + +STATIC_ASSERT_SIZEOF (iavf_vfint_icr0_ena1, 4); #endif /* _IIAVF_REGS_H_ */ diff --git a/src/plugins/dev_iavf/port.c b/src/plugins/dev_iavf/port.c index c0f363c3115..7888fcef0a6 100644 --- a/src/plugins/dev_iavf/port.c +++ b/src/plugins/dev_iavf/port.c @@ -24,16 +24,15 @@ static const u8 default_rss_key[] = { 0x55, 0x7d, 0x99, 0x58, 0x3a, 0xe1, 0x38, 0xc9, 0x2e, 0x81, 0x15, 0x03, 0x66, }; -const static iavf_dyn_ctln dyn_ctln_disabled = {}; -const static iavf_dyn_ctln dyn_ctln_enabled = { +const static iavf_dyn_ctl dyn_ctln_disabled = {}; +const static iavf_dyn_ctl dyn_ctln_enabled = { + .intena = 1, .clearpba = 1, .interval = IAVF_ITR_INT / 2, - .intena = 1, }; -const static iavf_dyn_ctln dyn_ctln_wb_on_itr = { - .clearpba = 1, +const static iavf_dyn_ctl dyn_ctln_wb_on_itr = { .itr_indx = 1, - .interval = 32 / 2, + .interval = 2, .wb_on_itr = 1, }; @@ -172,22 +171,30 @@ iavf_port_init_vsi_queues (vlib_main_t *vm, vnet_dev_port_t *port) } vnet_dev_rv_t -iavf_port_rx_irq_enable_disable (vlib_main_t *vm, vnet_dev_port_t *port, - int enable) +iavf_port_rx_irq_config (vlib_main_t *vm, vnet_dev_port_t *port, int enable) { vnet_dev_t *dev = port->dev; iavf_device_t *ad = vnet_dev_get_data (dev); iavf_port_t *ap = vnet_dev_get_port_data (port); - u16 n_threads = vlib_get_n_threads (); - u8 buffer[VIRTCHNL_MSG_SZ (virtchnl_irq_map_info_t, vecmap, n_threads)]; + u16 n_rx_vectors = ap->n_rx_vectors; + u8 buffer[VIRTCHNL_MSG_SZ (virtchnl_irq_map_info_t, vecmap, n_rx_vectors)]; + u8 n_intr_mode_queues_per_vector[n_rx_vectors]; + u8 n_queues_per_vector[n_rx_vectors]; virtchnl_irq_map_info_t *im = (virtchnl_irq_map_info_t *) buffer; vnet_dev_rv_t rv; + log_debug (dev, "intr mode per queue bitmap 0x%x", + ap->intr_mode_per_rxq_bitmap); + + for (u32 i = 0; i < n_rx_vectors; i++) + n_intr_mode_queues_per_vector[i] = n_queues_per_vector[i] = 0; + + *im = (virtchnl_irq_map_info_t){ + .num_vectors = n_rx_vectors, + }; + if (port->attr.caps.interrupt_mode) { - *im = (virtchnl_irq_map_info_t){ - .num_vectors = n_threads, - }; for (u16 i = 0; i < im->num_vectors; i++) im->vecmap[i] = (virtchnl_vector_map_t){ .vsi_id = ap->vsi_id, @@ -196,16 +203,19 @@ iavf_port_rx_irq_enable_disable (vlib_main_t *vm, vnet_dev_port_t *port, if (enable) foreach_vnet_dev_port_rx_queue (rxq, port) if (rxq->enabled) - im->vecmap[rxq->rx_thread_index].rxq_map |= 1 << rxq->queue_id; + { + u32 i = rxq->rx_thread_index; + im->vecmap[i].rxq_map |= 1 << rxq->queue_id; + n_queues_per_vector[i]++; + n_intr_mode_queues_per_vector[i] += + u64_is_bit_set (ap->intr_mode_per_rxq_bitmap, rxq->queue_id); + } } else { - *im = (virtchnl_irq_map_info_t){ - .num_vectors = 1, - .vecmap[0] = { - .vsi_id = ap->vsi_id, - .vector_id = 1, - }, + im->vecmap[0] = (virtchnl_vector_map_t){ + .vsi_id = ap->vsi_id, + .vector_id = 1, }; if (enable) foreach_vnet_dev_port_rx_queue (rxq, port) @@ -216,36 +226,64 @@ iavf_port_rx_irq_enable_disable (vlib_main_t *vm, vnet_dev_port_t *port, if ((rv = iavf_vc_op_config_irq_map (vm, dev, im))) return rv; - for (int i = 0; i < im->num_vectors; i++) + for (int i = 0; i < n_rx_vectors; i++) { u32 val; - if (enable == 0) + if (enable == 0 || n_queues_per_vector[i] == 0) val = dyn_ctln_disabled.as_u32; - else if (ap->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) + else if (ap->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR && + n_intr_mode_queues_per_vector[i] == 0) val = dyn_ctln_wb_on_itr.as_u32; else val = dyn_ctln_enabled.as_u32; - iavf_reg_write (ad, AVFINT_DYN_CTLN (im->vecmap[i].vector_id), val); + iavf_reg_write (ad, IAVF_VFINT_DYN_CTLN (i), val); + log_debug (dev, "VFINT_DYN_CTLN(%u) set to 0x%x", i, val); } return rv; } +static void +avf_msix_n_handler (vlib_main_t *vm, vnet_dev_t *dev, u16 line) +{ + iavf_device_t *ad = vnet_dev_get_data (dev); + vnet_dev_port_t *port = vnet_dev_get_port_by_id (dev, 0); + + line--; + + iavf_reg_write (ad, IAVF_VFINT_DYN_CTLN (line), dyn_ctln_enabled.as_u32); + vlib_node_set_interrupt_pending (vlib_get_main_by_index (line), + port->intf.rx_node_index); +} + vnet_dev_rv_t iavf_port_init (vlib_main_t *vm, vnet_dev_port_t *port) { + vnet_dev_t *dev = port->dev; + iavf_port_t *ap = vnet_dev_get_port_data (port); vnet_dev_rv_t rv; log_debug (port->dev, "port %u", port->port_id); + ap->intr_mode_per_rxq_bitmap = 0; + foreach_vnet_dev_port_rx_queue (q, port) + if (q->interrupt_mode) + u64_bit_set (&ap->intr_mode_per_rxq_bitmap, q->queue_id, 1); + if ((rv = iavf_port_vlan_strip_disable (vm, port))) return rv; if ((rv = iavf_port_init_rss (vm, port))) return rv; + vnet_dev_pci_msix_add_handler (vm, dev, &avf_msix_n_handler, 1, + ap->n_rx_vectors); + vnet_dev_pci_msix_enable (vm, dev, 1, ap->n_rx_vectors); + for (u32 i = 1; i < ap->n_rx_vectors; i++) + vnet_dev_pci_msix_set_polling_thread (vm, dev, i + 1, i); + if (port->dev->poll_stats) iavf_port_add_counters (vm, port); @@ -296,7 +334,7 @@ iavf_port_start (vlib_main_t *vm, vnet_dev_port_t *port) if ((rv = iavf_port_init_vsi_queues (vm, port))) goto done; - if ((rv = iavf_port_rx_irq_enable_disable (vm, port, /* enable */ 1))) + if ((rv = iavf_port_rx_irq_config (vm, port, /* enable */ 1))) goto done; if ((rv = iavf_enable_disable_queues (vm, port, 1))) @@ -322,7 +360,7 @@ iavf_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) log_debug (port->dev, "port %u", port->port_id); iavf_enable_disable_queues (vm, port, /* enable */ 0); - iavf_port_rx_irq_enable_disable (vm, port, /* disable */ 0); + iavf_port_rx_irq_config (vm, port, /* disable */ 0); if (port->dev->poll_stats) vnet_dev_poll_port_remove (vm, port, iavf_port_poll_stats); @@ -387,6 +425,57 @@ iavf_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, iavf_vc_op_del_eth_addr (vm, port->dev, &al); } +static vnet_dev_rv_t +iavf_port_cfg_rxq_int_mode_change (vlib_main_t *vm, vnet_dev_port_t *port, + u16 qid, u8 state, u8 all) +{ + vnet_dev_rv_t rv = VNET_DEV_OK; + iavf_port_t *ap = vnet_dev_get_port_data (port); + vnet_dev_t *dev = port->dev; + char *ed = state ? "ena" : "disa"; + char qstr[16]; + u64 old, new = 0; + + state = state != 0; + old = ap->intr_mode_per_rxq_bitmap; + + if (all) + { + snprintf (qstr, sizeof (qstr), "all queues"); + if (state) + foreach_vnet_dev_port_rx_queue (q, port) + u64_bit_set (&new, q->queue_id, 1); + } + else + { + snprintf (qstr, sizeof (qstr), "queue %u", qid); + new = old; + u64_bit_set (&new, qid, state); + } + + if (new == old) + { + log_warn (dev, "interrupt mode already %sbled on %s", ed, qstr); + return rv; + } + + ap->intr_mode_per_rxq_bitmap = new; + + if (port->started) + { + if ((rv = iavf_port_rx_irq_config (vm, port, 1))) + { + ap->intr_mode_per_rxq_bitmap = old; + log_err (dev, "failed to %sble interrupt mode on %s", ed, qstr); + return rv; + } + } + + log_debug (dev, "interrupt mode %sbled on %s, new bitmap is 0x%x", ed, qstr, + new); + return rv; +} + vnet_dev_rv_t iavf_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_cfg_change_req_t *req) @@ -434,6 +523,16 @@ iavf_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, case VNET_DEV_PORT_CFG_MAX_FRAME_SIZE: break; + case VNET_DEV_PORT_CFG_RXQ_INTR_MODE_ENABLE: + rv = iavf_port_cfg_rxq_int_mode_change (vm, port, req->queue_id, 1, + req->all_queues); + break; + + case VNET_DEV_PORT_CFG_RXQ_INTR_MODE_DISABLE: + rv = iavf_port_cfg_rxq_int_mode_change (vm, port, req->queue_id, 0, + req->all_queues); + break; + default: return VNET_DEV_ERR_NOT_SUPPORTED; }; diff --git a/src/plugins/dev_iavf/virtchnl.h b/src/plugins/dev_iavf/virtchnl.h index d1410129ba1..2099104c8ad 100644 --- a/src/plugins/dev_iavf/virtchnl.h +++ b/src/plugins/dev_iavf/virtchnl.h @@ -19,23 +19,23 @@ enum #undef _ }; -#define AVFINT_DYN_CTLN(x) (0x00003800 + (0x4 * x)) -#define AVFINT_ICR0 0x00004800 -#define AVFINT_ICR0_ENA1 0x00005000 -#define AVFINT_DYN_CTL0 0x00005C00 -#define IAVF_ARQBAH 0x00006000 -#define IAVF_ATQH 0x00006400 -#define IAVF_ATQLEN 0x00006800 -#define IAVF_ARQBAL 0x00006C00 -#define IAVF_ARQT 0x00007000 -#define IAVF_ARQH 0x00007400 -#define IAVF_ATQBAH 0x00007800 -#define IAVF_ATQBAL 0x00007C00 -#define IAVF_ARQLEN 0x00008000 -#define IAVF_ATQT 0x00008400 -#define VFGEN_RSTAT 0x00008800 -#define IAVF_QTX_TAIL(q) (0x00000000 + (0x4 * q)) -#define IAVF_QRX_TAIL(q) (0x00002000 + (0x4 * q)) +#define IAVF_VFINT_DYN_CTLN(x) (0x00003800 + (0x4 * x)) +#define IAVF_VFINT_ICR0 0x00004800 +#define IAVF_VFINT_ICR0_ENA1 0x00005000 +#define IAVF_VFINT_DYN_CTL0 0x00005C00 +#define IAVF_ARQBAH 0x00006000 +#define IAVF_ATQH 0x00006400 +#define IAVF_ATQLEN 0x00006800 +#define IAVF_ARQBAL 0x00006C00 +#define IAVF_ARQT 0x00007000 +#define IAVF_ARQH 0x00007400 +#define IAVF_ATQBAH 0x00007800 +#define IAVF_ATQBAL 0x00007C00 +#define IAVF_ARQLEN 0x00008000 +#define IAVF_ATQT 0x00008400 +#define IAVF_VFGEN_RSTAT 0x00008800 +#define IAVF_QTX_TAIL(q) (0x00000000 + (0x4 * q)) +#define IAVF_QRX_TAIL(q) (0x00002000 + (0x4 * q)) #define foreach_virtchnl_op \ _ (0, UNKNOWN) \ diff --git a/src/vppinfra/bitops.h b/src/vppinfra/bitops.h index 3d17f0d5ace..c1122f59ff6 100644 --- a/src/vppinfra/bitops.h +++ b/src/vppinfra/bitops.h @@ -315,6 +315,36 @@ bit_extract_u64 (u64 v, u64 mask) #endif } +static_always_inline void +u64_bit_set (u64 *p, u8 bit_index, u8 is_one) +{ + u64 val = *p; + val &= ~(1ULL << bit_index); + val |= 1ULL << bit_index; + *p = val; +} + +static_always_inline void +u32_bit_set (u32 *p, u8 bit_index, u8 is_one) +{ + u32 val = *p; + val &= ~(1U << bit_index); + val |= 1U << bit_index; + *p = val; +} + +static_always_inline int +u64_is_bit_set (u64 v, u8 bit_index) +{ + return (v & 1ULL << bit_index) != 0; +} + +static_always_inline int +u32_is_bit_set (u32 v, u8 bit_index) +{ + return (v & 1U << bit_index) != 0; +} + #else #warning "already included" #endif /* included_clib_bitops_h */ |