aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/net/cxgbe
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/cxgbe')
-rw-r--r--drivers/net/cxgbe/Makefile2
-rw-r--r--drivers/net/cxgbe/base/adapter.h4
-rw-r--r--drivers/net/cxgbe/base/common.h8
-rw-r--r--drivers/net/cxgbe/base/t4_hw.c108
-rw-r--r--drivers/net/cxgbe/base/t4_msg.h44
-rw-r--r--drivers/net/cxgbe/base/t4_regs.h8
-rw-r--r--drivers/net/cxgbe/base/t4_tcb.h5
-rw-r--r--drivers/net/cxgbe/base/t4fw_interface.h52
-rw-r--r--drivers/net/cxgbe/cxgbe.h15
-rw-r--r--drivers/net/cxgbe/cxgbe_ethdev.c35
-rw-r--r--drivers/net/cxgbe/cxgbe_filter.c117
-rw-r--r--drivers/net/cxgbe/cxgbe_filter.h35
-rw-r--r--drivers/net/cxgbe/cxgbe_flow.c287
-rw-r--r--drivers/net/cxgbe/cxgbe_flow.h1
-rw-r--r--drivers/net/cxgbe/cxgbe_main.c69
-rw-r--r--drivers/net/cxgbe/cxgbevf_main.c9
-rw-r--r--drivers/net/cxgbe/l2t.c227
-rw-r--r--drivers/net/cxgbe/l2t.h57
-rw-r--r--drivers/net/cxgbe/meson.build2
-rw-r--r--drivers/net/cxgbe/mps_tcam.c243
-rw-r--r--drivers/net/cxgbe/mps_tcam.h52
-rw-r--r--drivers/net/cxgbe/sge.c24
22 files changed, 1309 insertions, 95 deletions
diff --git a/drivers/net/cxgbe/Makefile b/drivers/net/cxgbe/Makefile
index 5d66c4b3..68466f13 100644
--- a/drivers/net/cxgbe/Makefile
+++ b/drivers/net/cxgbe/Makefile
@@ -53,6 +53,8 @@ SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_filter.c
SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_flow.c
SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += t4_hw.c
SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += clip_tbl.c
+SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += mps_tcam.c
+SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += l2t.c
SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += t4vf_hw.c
include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/cxgbe/base/adapter.h b/drivers/net/cxgbe/base/adapter.h
index e98dd218..47cfc5f5 100644
--- a/drivers/net/cxgbe/base/adapter.h
+++ b/drivers/net/cxgbe/base/adapter.h
@@ -324,7 +324,11 @@ struct adapter {
unsigned int clipt_start; /* CLIP table start */
unsigned int clipt_end; /* CLIP table end */
+ unsigned int l2t_start; /* Layer 2 table start */
+ unsigned int l2t_end; /* Layer 2 table end */
struct clip_tbl *clipt; /* CLIP table */
+ struct l2t_data *l2t; /* Layer 2 table */
+ struct mpstcam_table *mpstcam;
struct tid_info tids; /* Info used to access TID related tables */
};
diff --git a/drivers/net/cxgbe/base/common.h b/drivers/net/cxgbe/base/common.h
index 157201da..fd200668 100644
--- a/drivers/net/cxgbe/base/common.h
+++ b/drivers/net/cxgbe/base/common.h
@@ -157,6 +157,7 @@ struct tp_params {
int port_shift;
int protocol_shift;
int ethertype_shift;
+ int macmatch_shift;
u64 hash_filter_mask;
};
@@ -270,6 +271,7 @@ struct adapter_params {
bool ulptx_memwrite_dsgl; /* use of T5 DSGL allowed */
u8 fw_caps_support; /* 32-bit Port Capabilities */
+ u8 filter2_wr_support; /* FW support for FILTER2_WR */
};
/* Firmware Port Capabilities types.
@@ -388,6 +390,12 @@ int t4_free_vi(struct adapter *adap, unsigned int mbox,
int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
int mtu, int promisc, int all_multi, int bcast, int vlanex,
bool sleep_ok);
+int t4_free_raw_mac_filt(struct adapter *adap, unsigned int viid,
+ const u8 *addr, const u8 *mask, unsigned int idx,
+ u8 lookup_type, u8 port_id, bool sleep_ok);
+int t4_alloc_raw_mac_filt(struct adapter *adap, unsigned int viid,
+ const u8 *addr, const u8 *mask, unsigned int idx,
+ u8 lookup_type, u8 port_id, bool sleep_ok);
int t4_change_mac(struct adapter *adap, unsigned int mbox, unsigned int viid,
int idx, const u8 *addr, bool persist, bool add_smt);
int t4_enable_vi_params(struct adapter *adap, unsigned int mbox,
diff --git a/drivers/net/cxgbe/base/t4_hw.c b/drivers/net/cxgbe/base/t4_hw.c
index 31762c9c..701e0b1f 100644
--- a/drivers/net/cxgbe/base/t4_hw.c
+++ b/drivers/net/cxgbe/base/t4_hw.c
@@ -4162,6 +4162,112 @@ int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
}
/**
+ * t4_alloc_raw_mac_filt - Adds a raw mac entry in mps tcam
+ * @adap: the adapter
+ * @viid: the VI id
+ * @mac: the MAC address
+ * @mask: the mask
+ * @idx: index at which to add this entry
+ * @port_id: the port index
+ * @lookup_type: MAC address for inner (1) or outer (0) header
+ * @sleep_ok: call is allowed to sleep
+ *
+ * Adds the mac entry at the specified index using raw mac interface.
+ *
+ * Returns a negative error number or the allocated index for this mac.
+ */
+int t4_alloc_raw_mac_filt(struct adapter *adap, unsigned int viid,
+ const u8 *addr, const u8 *mask, unsigned int idx,
+ u8 lookup_type, u8 port_id, bool sleep_ok)
+{
+ int ret = 0;
+ struct fw_vi_mac_cmd c;
+ struct fw_vi_mac_raw *p = &c.u.raw;
+ u32 val;
+
+ memset(&c, 0, sizeof(c));
+ c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_VI_MAC_CMD) |
+ F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
+ V_FW_VI_MAC_CMD_VIID(viid));
+ val = V_FW_CMD_LEN16(1) |
+ V_FW_VI_MAC_CMD_ENTRY_TYPE(FW_VI_MAC_TYPE_RAW);
+ c.freemacs_to_len16 = cpu_to_be32(val);
+
+ /* Specify that this is an inner mac address */
+ p->raw_idx_pkd = cpu_to_be32(V_FW_VI_MAC_CMD_RAW_IDX(idx));
+
+ /* Lookup Type. Outer header: 0, Inner header: 1 */
+ p->data0_pkd = cpu_to_be32(V_DATALKPTYPE(lookup_type) |
+ V_DATAPORTNUM(port_id));
+ /* Lookup mask and port mask */
+ p->data0m_pkd = cpu_to_be64(V_DATALKPTYPE(M_DATALKPTYPE) |
+ V_DATAPORTNUM(M_DATAPORTNUM));
+
+ /* Copy the address and the mask */
+ memcpy((u8 *)&p->data1[0] + 2, addr, ETHER_ADDR_LEN);
+ memcpy((u8 *)&p->data1m[0] + 2, mask, ETHER_ADDR_LEN);
+
+ ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok);
+ if (ret == 0) {
+ ret = G_FW_VI_MAC_CMD_RAW_IDX(be32_to_cpu(p->raw_idx_pkd));
+ if (ret != (int)idx)
+ ret = -ENOMEM;
+ }
+
+ return ret;
+}
+
+/**
+ * t4_free_raw_mac_filt - Frees a raw mac entry in mps tcam
+ * @adap: the adapter
+ * @viid: the VI id
+ * @addr: the MAC address
+ * @mask: the mask
+ * @idx: index of the entry in mps tcam
+ * @lookup_type: MAC address for inner (1) or outer (0) header
+ * @port_id: the port index
+ * @sleep_ok: call is allowed to sleep
+ *
+ * Removes the mac entry at the specified index using raw mac interface.
+ *
+ * Returns a negative error number on failure.
+ */
+int t4_free_raw_mac_filt(struct adapter *adap, unsigned int viid,
+ const u8 *addr, const u8 *mask, unsigned int idx,
+ u8 lookup_type, u8 port_id, bool sleep_ok)
+{
+ struct fw_vi_mac_cmd c;
+ struct fw_vi_mac_raw *p = &c.u.raw;
+ u32 raw;
+
+ memset(&c, 0, sizeof(c));
+ c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_VI_MAC_CMD) |
+ F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
+ V_FW_CMD_EXEC(0) |
+ V_FW_VI_MAC_CMD_VIID(viid));
+ raw = V_FW_VI_MAC_CMD_ENTRY_TYPE(FW_VI_MAC_TYPE_RAW);
+ c.freemacs_to_len16 = cpu_to_be32(V_FW_VI_MAC_CMD_FREEMACS(0) |
+ raw |
+ V_FW_CMD_LEN16(1));
+
+ p->raw_idx_pkd = cpu_to_be32(V_FW_VI_MAC_CMD_RAW_IDX(idx) |
+ FW_VI_MAC_ID_BASED_FREE);
+
+ /* Lookup Type. Outer header: 0, Inner header: 1 */
+ p->data0_pkd = cpu_to_be32(V_DATALKPTYPE(lookup_type) |
+ V_DATAPORTNUM(port_id));
+ /* Lookup mask and port mask */
+ p->data0m_pkd = cpu_to_be64(V_DATALKPTYPE(M_DATALKPTYPE) |
+ V_DATAPORTNUM(M_DATAPORTNUM));
+
+ /* Copy the address and the mask */
+ memcpy((u8 *)&p->data1[0] + 2, addr, ETHER_ADDR_LEN);
+ memcpy((u8 *)&p->data1m[0] + 2, mask, ETHER_ADDR_LEN);
+
+ return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok);
+}
+
+/**
* t4_change_mac - modifies the exact-match filter for a MAC address
* @adap: the adapter
* @mbox: mailbox to use for the FW command
@@ -5145,6 +5251,8 @@ int t4_init_tp_params(struct adapter *adap)
F_PROTOCOL);
adap->params.tp.ethertype_shift = t4_filter_field_shift(adap,
F_ETHERTYPE);
+ adap->params.tp.macmatch_shift = t4_filter_field_shift(adap,
+ F_MACMATCH);
/*
* If TP_INGRESS_CONFIG.VNID == 0, then TP_VLAN_PRI_MAP.VNIC_ID
diff --git a/drivers/net/cxgbe/base/t4_msg.h b/drivers/net/cxgbe/base/t4_msg.h
index 5d433c91..9e052b0f 100644
--- a/drivers/net/cxgbe/base/t4_msg.h
+++ b/drivers/net/cxgbe/base/t4_msg.h
@@ -11,7 +11,9 @@ enum {
CPL_SET_TCB_FIELD = 0x5,
CPL_ABORT_REQ = 0xA,
CPL_ABORT_RPL = 0xB,
+ CPL_L2T_WRITE_REQ = 0x12,
CPL_TID_RELEASE = 0x1A,
+ CPL_L2T_WRITE_RPL = 0x23,
CPL_ACT_OPEN_RPL = 0x25,
CPL_ABORT_RPL_RSS = 0x2D,
CPL_SET_TCB_RPL = 0x3A,
@@ -30,6 +32,7 @@ enum CPL_error {
enum {
ULP_MODE_NONE = 0,
+ ULP_MODE_TCPDDP = 5,
};
enum {
@@ -66,6 +69,9 @@ union opcode_tid {
#define M_TID_TID 0x3fff
#define G_TID_TID(x) (((x) >> S_TID_TID) & M_TID_TID)
+#define S_TID_QID 14
+#define V_TID_QID(x) ((x) << S_TID_QID)
+
struct rss_header {
__u8 opcode;
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
@@ -133,6 +139,12 @@ struct work_request_hdr {
#define V_TCAM_BYPASS(x) ((__u64)(x) << S_TCAM_BYPASS)
#define F_TCAM_BYPASS V_TCAM_BYPASS(1ULL)
+#define S_L2T_IDX 36
+#define V_L2T_IDX(x) ((__u64)(x) << S_L2T_IDX)
+
+#define S_NAGLE 49
+#define V_NAGLE(x) ((__u64)(x) << S_NAGLE)
+
/* option 2 fields */
#define S_RSS_QUEUE 0
#define V_RSS_QUEUE(x) ((x) << S_RSS_QUEUE)
@@ -151,6 +163,9 @@ struct work_request_hdr {
#define S_CCTRL_ECN 27
#define V_CCTRL_ECN(x) ((x) << S_CCTRL_ECN)
+#define S_SACK_EN 30
+#define V_SACK_EN(x) ((x) << S_SACK_EN)
+
#define S_T5_OPT_2_VALID 31
#define V_T5_OPT_2_VALID(x) ((x) << S_T5_OPT_2_VALID)
#define F_T5_OPT_2_VALID V_T5_OPT_2_VALID(1U)
@@ -421,6 +436,35 @@ struct cpl_rx_pkt {
__be16 err_vec;
};
+struct cpl_l2t_write_req {
+ WR_HDR;
+ union opcode_tid ot;
+ __be16 params;
+ __be16 l2t_idx;
+ __be16 vlan;
+ __u8 dst_mac[6];
+};
+
+/* cpl_l2t_write_req.params fields */
+#define S_L2T_W_PORT 8
+#define V_L2T_W_PORT(x) ((x) << S_L2T_W_PORT)
+
+#define S_L2T_W_LPBK 10
+#define V_L2T_W_LPBK(x) ((x) << S_L2T_W_LPBK)
+
+#define S_L2T_W_ARPMISS 11
+#define V_L2T_W_ARPMISS(x) ((x) << S_L2T_W_ARPMISS)
+
+#define S_L2T_W_NOREPLY 15
+#define V_L2T_W_NOREPLY(x) ((x) << S_L2T_W_NOREPLY)
+
+struct cpl_l2t_write_rpl {
+ RSS_HDR
+ union opcode_tid ot;
+ __u8 status;
+ __u8 rsvd[3];
+};
+
/* rx_pkt.l2info fields */
#define S_RXF_UDP 22
#define V_RXF_UDP(x) ((x) << S_RXF_UDP)
diff --git a/drivers/net/cxgbe/base/t4_regs.h b/drivers/net/cxgbe/base/t4_regs.h
index 6f872edc..af8c741e 100644
--- a/drivers/net/cxgbe/base/t4_regs.h
+++ b/drivers/net/cxgbe/base/t4_regs.h
@@ -45,6 +45,14 @@
#define MPS_T5_CLS_SRAM_H(idx) (A_MPS_T5_CLS_SRAM_H + (idx) * 8)
#define NUM_MPS_T5_CLS_SRAM_H_INSTANCES 512
+#define S_DATAPORTNUM 12
+#define M_DATAPORTNUM 0xfU
+#define V_DATAPORTNUM(x) ((x) << S_DATAPORTNUM)
+
+#define S_DATALKPTYPE 10
+#define M_DATALKPTYPE 0x3U
+#define V_DATALKPTYPE(x) ((x) << S_DATALKPTYPE)
+
/* registers for module SGE */
#define SGE_BASE_ADDR 0x1000
diff --git a/drivers/net/cxgbe/base/t4_tcb.h b/drivers/net/cxgbe/base/t4_tcb.h
index 25435f9f..68cda773 100644
--- a/drivers/net/cxgbe/base/t4_tcb.h
+++ b/drivers/net/cxgbe/base/t4_tcb.h
@@ -6,6 +6,9 @@
#ifndef _T4_TCB_DEFS_H
#define _T4_TCB_DEFS_H
+/* 95:32 */
+#define W_TCB_T_FLAGS 1
+
/* 105:96 */
#define W_TCB_RSS_INFO 3
#define S_TCB_RSS_INFO 0
@@ -23,4 +26,6 @@
#define M_TCB_T_RTT_TS_RECENT_AGE 0xffffffffULL
#define V_TCB_T_RTT_TS_RECENT_AGE(x) ((x) << S_TCB_T_RTT_TS_RECENT_AGE)
+#define S_TF_CCTRL_RFR 62
+
#endif /* _T4_TCB_DEFS_H */
diff --git a/drivers/net/cxgbe/base/t4fw_interface.h b/drivers/net/cxgbe/base/t4fw_interface.h
index e80b58a3..06d3ef3a 100644
--- a/drivers/net/cxgbe/base/t4fw_interface.h
+++ b/drivers/net/cxgbe/base/t4fw_interface.h
@@ -61,6 +61,7 @@ enum fw_wr_opcodes {
FW_ETH_TX_PKTS_WR = 0x09,
FW_ETH_TX_PKT_VM_WR = 0x11,
FW_ETH_TX_PKTS_VM_WR = 0x12,
+ FW_FILTER2_WR = 0x77,
FW_ETH_TX_PKTS2_WR = 0x78,
};
@@ -165,7 +166,7 @@ enum fw_filter_wr_cookie {
FW_FILTER_WR_EINVAL,
};
-struct fw_filter_wr {
+struct fw_filter2_wr {
__be32 op_pkd;
__be32 len16_pkd;
__be64 r3;
@@ -195,6 +196,19 @@ struct fw_filter_wr {
__be16 fpm;
__be16 r7;
__u8 sma[6];
+ __be16 r8;
+ __u8 filter_type_swapmac;
+ __u8 natmode_to_ulp_type;
+ __be16 newlport;
+ __be16 newfport;
+ __u8 newlip[16];
+ __u8 newfip[16];
+ __be32 natseqcheck;
+ __be32 r9;
+ __be64 r10;
+ __be64 r11;
+ __be64 r12;
+ __be64 r13;
};
#define S_FW_FILTER_WR_TID 12
@@ -300,6 +314,15 @@ struct fw_filter_wr {
#define S_FW_FILTER_WR_MATCHTYPEM 0
#define V_FW_FILTER_WR_MATCHTYPEM(x) ((x) << S_FW_FILTER_WR_MATCHTYPEM)
+#define S_FW_FILTER2_WR_SWAPMAC 0
+#define V_FW_FILTER2_WR_SWAPMAC(x) ((x) << S_FW_FILTER2_WR_SWAPMAC)
+
+#define S_FW_FILTER2_WR_NATMODE 5
+#define V_FW_FILTER2_WR_NATMODE(x) ((x) << S_FW_FILTER2_WR_NATMODE)
+
+#define S_FW_FILTER2_WR_ULP_TYPE 0
+#define V_FW_FILTER2_WR_ULP_TYPE(x) ((x) << S_FW_FILTER2_WR_ULP_TYPE)
+
/******************************************************************************
* C O M M A N D s
*********************/
@@ -655,6 +678,7 @@ enum fw_params_param_dev {
FW_PARAMS_PARAM_DEV_FWREV = 0x0B, /* fw version */
FW_PARAMS_PARAM_DEV_TPREV = 0x0C, /* tp version */
FW_PARAMS_PARAM_DEV_ULPTX_MEMWRITE_DSGL = 0x17,
+ FW_PARAMS_PARAM_DEV_FILTER2_WR = 0x1D,
};
/*
@@ -665,6 +689,8 @@ enum fw_params_param_pfvf {
FW_PARAMS_PARAM_PFVF_CLIP_END = 0x04,
FW_PARAMS_PARAM_PFVF_FILTER_START = 0x05,
FW_PARAMS_PARAM_PFVF_FILTER_END = 0x06,
+ FW_PARAMS_PARAM_PFVF_L2T_START = 0x13,
+ FW_PARAMS_PARAM_PFVF_L2T_END = 0x14,
FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP = 0x31,
FW_PARAMS_PARAM_PFVF_PORT_CAPS32 = 0x3A
};
@@ -1280,12 +1306,17 @@ struct fw_vi_cmd {
/* Special VI_MAC command index ids */
#define FW_VI_MAC_ADD_MAC 0x3FF
#define FW_VI_MAC_ADD_PERSIST_MAC 0x3FE
+#define FW_VI_MAC_ID_BASED_FREE 0x3FC
enum fw_vi_mac_smac {
FW_VI_MAC_MPS_TCAM_ENTRY,
FW_VI_MAC_SMT_AND_MPSTCAM
};
+enum fw_vi_mac_entry_types {
+ FW_VI_MAC_TYPE_RAW = 0x2,
+};
+
struct fw_vi_mac_cmd {
__be32 op_to_viid;
__be32 freemacs_to_len16;
@@ -1297,6 +1328,13 @@ struct fw_vi_mac_cmd {
struct fw_vi_mac_hash {
__be64 hashvec;
} hash;
+ struct fw_vi_mac_raw {
+ __be32 raw_idx_pkd;
+ __be32 data0_pkd;
+ __be32 data1[2];
+ __be64 data0m_pkd;
+ __be32 data1m[2];
+ } raw;
} u;
};
@@ -1306,6 +1344,12 @@ struct fw_vi_mac_cmd {
#define G_FW_VI_MAC_CMD_VIID(x) \
(((x) >> S_FW_VI_MAC_CMD_VIID) & M_FW_VI_MAC_CMD_VIID)
+#define S_FW_VI_MAC_CMD_FREEMACS 31
+#define V_FW_VI_MAC_CMD_FREEMACS(x) ((x) << S_FW_VI_MAC_CMD_FREEMACS)
+
+#define S_FW_VI_MAC_CMD_ENTRY_TYPE 23
+#define V_FW_VI_MAC_CMD_ENTRY_TYPE(x) ((x) << S_FW_VI_MAC_CMD_ENTRY_TYPE)
+
#define S_FW_VI_MAC_CMD_VALID 15
#define M_FW_VI_MAC_CMD_VALID 0x1
#define V_FW_VI_MAC_CMD_VALID(x) ((x) << S_FW_VI_MAC_CMD_VALID)
@@ -1325,6 +1369,12 @@ struct fw_vi_mac_cmd {
#define G_FW_VI_MAC_CMD_IDX(x) \
(((x) >> S_FW_VI_MAC_CMD_IDX) & M_FW_VI_MAC_CMD_IDX)
+#define S_FW_VI_MAC_CMD_RAW_IDX 16
+#define M_FW_VI_MAC_CMD_RAW_IDX 0xffff
+#define V_FW_VI_MAC_CMD_RAW_IDX(x) ((x) << S_FW_VI_MAC_CMD_RAW_IDX)
+#define G_FW_VI_MAC_CMD_RAW_IDX(x) \
+ (((x) >> S_FW_VI_MAC_CMD_RAW_IDX) & M_FW_VI_MAC_CMD_RAW_IDX)
+
struct fw_vi_rxmode_cmd {
__be32 op_to_viid;
__be32 retval_len16;
diff --git a/drivers/net/cxgbe/cxgbe.h b/drivers/net/cxgbe/cxgbe.h
index 5e6f5c98..eb58f880 100644
--- a/drivers/net/cxgbe/cxgbe.h
+++ b/drivers/net/cxgbe/cxgbe.h
@@ -34,6 +34,21 @@
ETH_RSS_IPV6_UDP_EX)
#define CXGBE_RSS_HF_ALL (ETH_RSS_IP | ETH_RSS_TCP | ETH_RSS_UDP)
+/* Tx/Rx Offloads supported */
+#define CXGBE_TX_OFFLOADS (DEV_TX_OFFLOAD_VLAN_INSERT | \
+ DEV_TX_OFFLOAD_IPV4_CKSUM | \
+ DEV_TX_OFFLOAD_UDP_CKSUM | \
+ DEV_TX_OFFLOAD_TCP_CKSUM | \
+ DEV_TX_OFFLOAD_TCP_TSO)
+
+#define CXGBE_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_STRIP | \
+ DEV_RX_OFFLOAD_IPV4_CKSUM | \
+ DEV_RX_OFFLOAD_UDP_CKSUM | \
+ DEV_RX_OFFLOAD_TCP_CKSUM | \
+ DEV_RX_OFFLOAD_JUMBO_FRAME | \
+ DEV_RX_OFFLOAD_SCATTER)
+
+
#define CXGBE_DEVARG_KEEP_OVLAN "keep_ovlan"
#define CXGBE_DEVARG_FORCE_LINK_UP "force_link_up"
diff --git a/drivers/net/cxgbe/cxgbe_ethdev.c b/drivers/net/cxgbe/cxgbe_ethdev.c
index 4dcad7a2..b2f83ea3 100644
--- a/drivers/net/cxgbe/cxgbe_ethdev.c
+++ b/drivers/net/cxgbe/cxgbe_ethdev.c
@@ -59,19 +59,6 @@
*/
#include "t4_pci_id_tbl.h"
-#define CXGBE_TX_OFFLOADS (DEV_TX_OFFLOAD_VLAN_INSERT |\
- DEV_TX_OFFLOAD_IPV4_CKSUM |\
- DEV_TX_OFFLOAD_UDP_CKSUM |\
- DEV_TX_OFFLOAD_TCP_CKSUM |\
- DEV_TX_OFFLOAD_TCP_TSO)
-
-#define CXGBE_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_STRIP |\
- DEV_RX_OFFLOAD_CRC_STRIP |\
- DEV_RX_OFFLOAD_IPV4_CKSUM |\
- DEV_RX_OFFLOAD_JUMBO_FRAME |\
- DEV_RX_OFFLOAD_UDP_CKSUM |\
- DEV_RX_OFFLOAD_TCP_CKSUM)
-
uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
@@ -341,6 +328,7 @@ void cxgbe_dev_close(struct rte_eth_dev *eth_dev)
int cxgbe_dev_start(struct rte_eth_dev *eth_dev)
{
struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+ struct rte_eth_rxmode *rx_conf = &eth_dev->data->dev_conf.rxmode;
struct adapter *adapter = pi->adapter;
int err = 0, i;
@@ -361,6 +349,11 @@ int cxgbe_dev_start(struct rte_eth_dev *eth_dev)
goto out;
}
+ if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)
+ eth_dev->data->scattered_rx = 1;
+ else
+ eth_dev->data->scattered_rx = 0;
+
cxgbe_enable_rx_queues(pi);
err = setup_rss(pi);
@@ -407,26 +400,16 @@ void cxgbe_dev_stop(struct rte_eth_dev *eth_dev)
* have been disabled
*/
t4_sge_eth_clear_queues(pi);
+ eth_dev->data->scattered_rx = 0;
}
int cxgbe_dev_configure(struct rte_eth_dev *eth_dev)
{
struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
struct adapter *adapter = pi->adapter;
- uint64_t configured_offloads;
int err;
CXGBE_FUNC_TRACE();
- configured_offloads = eth_dev->data->dev_conf.rxmode.offloads;
-
- /* KEEP_CRC offload flag is not supported by PMD
- * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
- */
- if (rte_eth_dev_must_keep_crc(configured_offloads)) {
- dev_info(adapter, "can't disable hw crc strip\n");
- eth_dev->data->dev_conf.rxmode.offloads |=
- DEV_RX_OFFLOAD_CRC_STRIP;
- }
if (!(adapter->flags & FW_QUEUE_BOUND)) {
err = setup_sge_fwevtq(adapter);
@@ -1075,11 +1058,9 @@ static int cxgbe_get_regs(struct rte_eth_dev *eth_dev,
int cxgbe_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr)
{
struct port_info *pi = (struct port_info *)(dev->data->dev_private);
- struct adapter *adapter = pi->adapter;
int ret;
- ret = t4_change_mac(adapter, adapter->mbox, pi->viid,
- pi->xact_addr_filt, (u8 *)addr, true, true);
+ ret = cxgbe_mpstcam_modify(pi, (int)pi->xact_addr_filt, (u8 *)addr);
if (ret < 0) {
dev_err(adapter, "failed to set mac addr; err = %d\n",
ret);
diff --git a/drivers/net/cxgbe/cxgbe_filter.c b/drivers/net/cxgbe/cxgbe_filter.c
index 7f0d3800..ef1102be 100644
--- a/drivers/net/cxgbe/cxgbe_filter.c
+++ b/drivers/net/cxgbe/cxgbe_filter.c
@@ -8,6 +8,7 @@
#include "t4_regs.h"
#include "cxgbe_filter.h"
#include "clip_tbl.h"
+#include "l2t.h"
/**
* Initialize Hash Filters
@@ -65,7 +66,8 @@ int validate_filter(struct adapter *adapter, struct ch_filter_specification *fs)
#define U(_mask, _field) \
(!(fconf & (_mask)) && S(_field))
- if (U(F_PORT, iport) || U(F_ETHERTYPE, ethtype) || U(F_PROTOCOL, proto))
+ if (U(F_PORT, iport) || U(F_ETHERTYPE, ethtype) ||
+ U(F_PROTOCOL, proto) || U(F_MACMATCH, macidx))
return -EOPNOTSUPP;
#undef S
@@ -87,6 +89,12 @@ int validate_filter(struct adapter *adapter, struct ch_filter_specification *fs)
if (fs->val.iport >= adapter->params.nports)
return -ERANGE;
+ if (!fs->cap && fs->nat_mode && !adapter->params.filter2_wr_support)
+ return -EOPNOTSUPP;
+
+ if (!fs->cap && fs->swapmac && !adapter->params.filter2_wr_support)
+ return -EOPNOTSUPP;
+
return 0;
}
@@ -165,6 +173,16 @@ static void set_tcb_field(struct adapter *adapter, unsigned int ftid,
}
/**
+ * Set one of the t_flags bits in the TCB.
+ */
+static void set_tcb_tflag(struct adapter *adap, unsigned int ftid,
+ unsigned int bit_pos, unsigned int val, int no_reply)
+{
+ set_tcb_field(adap, ftid, W_TCB_T_FLAGS, 1ULL << bit_pos,
+ (unsigned long long)val << bit_pos, no_reply);
+}
+
+/**
* Build a CPL_SET_TCB_FIELD message as payload of a ULP_TX_PKT command.
*/
static inline void mk_set_tcb_field_ulp(struct filter_entry *f,
@@ -257,6 +275,8 @@ static u64 hash_filter_ntuple(const struct filter_entry *f)
if (tp->ethertype_shift >= 0 && f->fs.mask.ethtype)
ntuple |= (u64)(f->fs.val.ethtype) << tp->ethertype_shift;
+ if (tp->macmatch_shift >= 0 && f->fs.mask.macidx)
+ ntuple |= (u64)(f->fs.val.macidx) << tp->macmatch_shift;
if (ntuple != tp->hash_filter_mask)
return 0;
@@ -425,7 +445,10 @@ static void mk_act_open_req6(struct filter_entry *f, struct rte_mbuf *mbuf,
req->local_ip_lo = local_lo;
req->peer_ip_hi = peer_hi;
req->peer_ip_lo = peer_lo;
- req->opt0 = cpu_to_be64(V_DELACK(f->fs.hitcnts) |
+ req->opt0 = cpu_to_be64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
+ f->fs.newvlan == VLAN_REWRITE) |
+ V_DELACK(f->fs.hitcnts) |
+ V_L2T_IDX(f->l2t ? f->l2t->idx : 0) |
V_SMAC_SEL((cxgbe_port_viid(f->dev) & 0x7F)
<< 1) |
V_TX_CHAN(f->fs.eport) |
@@ -436,6 +459,7 @@ static void mk_act_open_req6(struct filter_entry *f, struct rte_mbuf *mbuf,
V_RSS_QUEUE(f->fs.iq) |
F_T5_OPT_2_VALID |
F_RX_CHANNEL |
+ V_SACK_EN(f->fs.swapmac) |
V_CONG_CNTRL((f->fs.action == FILTER_DROP) |
(f->fs.dirsteer << 1)) |
V_CCTRL_ECN(f->fs.action == FILTER_SWITCH));
@@ -468,7 +492,10 @@ static void mk_act_open_req(struct filter_entry *f, struct rte_mbuf *mbuf,
f->fs.val.lip[2] << 16 | f->fs.val.lip[3] << 24;
req->peer_ip = f->fs.val.fip[0] | f->fs.val.fip[1] << 8 |
f->fs.val.fip[2] << 16 | f->fs.val.fip[3] << 24;
- req->opt0 = cpu_to_be64(V_DELACK(f->fs.hitcnts) |
+ req->opt0 = cpu_to_be64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
+ f->fs.newvlan == VLAN_REWRITE) |
+ V_DELACK(f->fs.hitcnts) |
+ V_L2T_IDX(f->l2t ? f->l2t->idx : 0) |
V_SMAC_SEL((cxgbe_port_viid(f->dev) & 0x7F)
<< 1) |
V_TX_CHAN(f->fs.eport) |
@@ -479,6 +506,7 @@ static void mk_act_open_req(struct filter_entry *f, struct rte_mbuf *mbuf,
V_RSS_QUEUE(f->fs.iq) |
F_T5_OPT_2_VALID |
F_RX_CHANNEL |
+ V_SACK_EN(f->fs.swapmac) |
V_CONG_CNTRL((f->fs.action == FILTER_DROP) |
(f->fs.dirsteer << 1)) |
V_CCTRL_ECN(f->fs.action == FILTER_SWITCH));
@@ -518,6 +546,22 @@ static int cxgbe_set_hash_filter(struct rte_eth_dev *dev,
f->dev = dev;
f->fs.iq = iq;
+ /*
+ * If the new filter requires loopback Destination MAC and/or VLAN
+ * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
+ * the filter.
+ */
+ if (f->fs.newvlan == VLAN_INSERT ||
+ f->fs.newvlan == VLAN_REWRITE) {
+ /* allocate L2T entry for new filter */
+ f->l2t = cxgbe_l2t_alloc_switching(dev, f->fs.vlan,
+ f->fs.eport, f->fs.dmac);
+ if (!f->l2t) {
+ ret = -ENOMEM;
+ goto out_err;
+ }
+ }
+
atid = cxgbe_alloc_atid(t, f);
if (atid < 0)
goto out_err;
@@ -591,6 +635,7 @@ void clear_filter(struct filter_entry *f)
/**
* t4_mk_filtdelwr - create a delete filter WR
+ * @adap: adapter context
* @ftid: the filter ID
* @wr: the filter work request to populate
* @qid: ingress queue to receive the delete notification
@@ -598,10 +643,14 @@ void clear_filter(struct filter_entry *f)
* Creates a filter work request to delete the supplied filter. If @qid is
* negative the delete notification is suppressed.
*/
-static void t4_mk_filtdelwr(unsigned int ftid, struct fw_filter_wr *wr, int qid)
+static void t4_mk_filtdelwr(struct adapter *adap, unsigned int ftid,
+ struct fw_filter2_wr *wr, int qid)
{
memset(wr, 0, sizeof(*wr));
- wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER_WR));
+ if (adap->params.filter2_wr_support)
+ wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER2_WR));
+ else
+ wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER_WR));
wr->len16_pkd = cpu_to_be32(V_FW_WR_LEN16(sizeof(*wr) / 16));
wr->tid_to_iq = cpu_to_be32(V_FW_FILTER_WR_TID(ftid) |
V_FW_FILTER_WR_NOREPLY(qid < 0));
@@ -619,7 +668,7 @@ static int del_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
struct adapter *adapter = ethdev2adap(dev);
struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
struct rte_mbuf *mbuf;
- struct fw_filter_wr *fwr;
+ struct fw_filter2_wr *fwr;
struct sge_ctrl_txq *ctrlq;
unsigned int port_id = ethdev2pinfo(dev)->port_id;
@@ -631,8 +680,8 @@ static int del_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
mbuf->data_len = sizeof(*fwr);
mbuf->pkt_len = mbuf->data_len;
- fwr = rte_pktmbuf_mtod(mbuf, struct fw_filter_wr *);
- t4_mk_filtdelwr(f->tid, fwr, adapter->sge.fw_evtq.abs_id);
+ fwr = rte_pktmbuf_mtod(mbuf, struct fw_filter2_wr *);
+ t4_mk_filtdelwr(adapter, f->tid, fwr, adapter->sge.fw_evtq.abs_id);
/*
* Mark the filter as "pending" and ship off the Filter Work Request.
@@ -648,11 +697,24 @@ int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
struct adapter *adapter = ethdev2adap(dev);
struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
struct rte_mbuf *mbuf;
- struct fw_filter_wr *fwr;
+ struct fw_filter2_wr *fwr;
struct sge_ctrl_txq *ctrlq;
unsigned int port_id = ethdev2pinfo(dev)->port_id;
int ret;
+ /*
+ * If the new filter requires loopback Destination MAC and/or VLAN
+ * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
+ * the filter.
+ */
+ if (f->fs.newvlan) {
+ /* allocate L2T entry for new filter */
+ f->l2t = cxgbe_l2t_alloc_switching(f->dev, f->fs.vlan,
+ f->fs.eport, f->fs.dmac);
+ if (!f->l2t)
+ return -ENOMEM;
+ }
+
ctrlq = &adapter->sge.ctrlq[port_id];
mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool);
if (!mbuf) {
@@ -663,13 +725,16 @@ int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
mbuf->data_len = sizeof(*fwr);
mbuf->pkt_len = mbuf->data_len;
- fwr = rte_pktmbuf_mtod(mbuf, struct fw_filter_wr *);
+ fwr = rte_pktmbuf_mtod(mbuf, struct fw_filter2_wr *);
memset(fwr, 0, sizeof(*fwr));
/*
* Construct the work request to set the filter.
*/
- fwr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER_WR));
+ if (adapter->params.filter2_wr_support)
+ fwr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER2_WR));
+ else
+ fwr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER_WR));
fwr->len16_pkd = cpu_to_be32(V_FW_WR_LEN16(sizeof(*fwr) / 16));
fwr->tid_to_iq =
cpu_to_be32(V_FW_FILTER_WR_TID(f->tid) |
@@ -680,9 +745,16 @@ int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
cpu_to_be32(V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
+ V_FW_FILTER_WR_INSVLAN
+ (f->fs.newvlan == VLAN_INSERT ||
+ f->fs.newvlan == VLAN_REWRITE) |
+ V_FW_FILTER_WR_RMVLAN
+ (f->fs.newvlan == VLAN_REMOVE ||
+ f->fs.newvlan == VLAN_REWRITE) |
V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
- V_FW_FILTER_WR_PRIO(f->fs.prio));
+ V_FW_FILTER_WR_PRIO(f->fs.prio) |
+ V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
fwr->ethtype = cpu_to_be16(f->fs.val.ethtype);
fwr->ethtypem = cpu_to_be16(f->fs.mask.ethtype);
fwr->smac_sel = 0;
@@ -691,7 +763,9 @@ int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
V_FW_FILTER_WR_RX_RPL_IQ(adapter->sge.fw_evtq.abs_id
));
fwr->maci_to_matchtypem =
- cpu_to_be32(V_FW_FILTER_WR_PORT(f->fs.val.iport) |
+ cpu_to_be32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
+ V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
+ V_FW_FILTER_WR_PORT(f->fs.val.iport) |
V_FW_FILTER_WR_PORTM(f->fs.mask.iport));
fwr->ptcl = f->fs.val.proto;
fwr->ptclm = f->fs.mask.proto;
@@ -704,6 +778,20 @@ int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
fwr->fp = cpu_to_be16(f->fs.val.fport);
fwr->fpm = cpu_to_be16(f->fs.mask.fport);
+ if (adapter->params.filter2_wr_support) {
+ fwr->filter_type_swapmac =
+ V_FW_FILTER2_WR_SWAPMAC(f->fs.swapmac);
+ fwr->natmode_to_ulp_type =
+ V_FW_FILTER2_WR_ULP_TYPE(f->fs.nat_mode ?
+ ULP_MODE_TCPDDP :
+ ULP_MODE_NONE) |
+ V_FW_FILTER2_WR_NATMODE(f->fs.nat_mode);
+ memcpy(fwr->newlip, f->fs.nat_lip, sizeof(fwr->newlip));
+ memcpy(fwr->newfip, f->fs.nat_fip, sizeof(fwr->newfip));
+ fwr->newlport = cpu_to_be16(f->fs.nat_lport);
+ fwr->newfport = cpu_to_be16(f->fs.nat_fport);
+ }
+
/*
* Mark the filter as "pending" and ship off the Filter Work Request.
* When we get the Work Request Reply we'll clear the pending status.
@@ -1046,6 +1134,9 @@ void hash_filter_rpl(struct adapter *adap, const struct cpl_act_open_rpl *rpl)
V_TCB_TIMESTAMP(0ULL) |
V_TCB_T_RTT_TS_RECENT_AGE(0ULL),
1);
+ if (f->fs.newvlan == VLAN_INSERT ||
+ f->fs.newvlan == VLAN_REWRITE)
+ set_tcb_tflag(adap, tid, S_TF_CCTRL_RFR, 1, 1);
break;
}
default:
diff --git a/drivers/net/cxgbe/cxgbe_filter.h b/drivers/net/cxgbe/cxgbe_filter.h
index af8fa752..b7bcbf56 100644
--- a/drivers/net/cxgbe/cxgbe_filter.h
+++ b/drivers/net/cxgbe/cxgbe_filter.h
@@ -77,6 +77,7 @@ struct ch_filter_tuple {
* Filter specification
*/
struct ch_filter_specification {
+ void *private;
/* Administrative fields for filter. */
uint32_t hitcnts:1; /* count filter hits in TCB */
uint32_t prio:1; /* filter has priority over active/server */
@@ -99,6 +100,22 @@ struct ch_filter_specification {
uint32_t iq:10; /* ingress queue */
uint32_t eport:2; /* egress port to switch packet out */
+ uint32_t swapmac:1; /* swap SMAC/DMAC for loopback packet */
+ uint32_t newvlan:2; /* rewrite VLAN Tag */
+ uint8_t dmac[ETHER_ADDR_LEN]; /* new destination MAC address */
+ uint16_t vlan; /* VLAN Tag to insert */
+
+ /*
+ * Switch proxy/rewrite fields. An ingress packet which matches a
+ * filter with "switch" set will be looped back out as an egress
+ * packet -- potentially with some header rewriting.
+ */
+ uint32_t nat_mode:3; /* specify NAT operation mode */
+
+ uint8_t nat_lip[16]; /* local IP to use after NAT'ing */
+ uint8_t nat_fip[16]; /* foreign IP to use after NAT'ing */
+ uint16_t nat_lport; /* local port number to use after NAT'ing */
+ uint16_t nat_fport; /* foreign port number to use after NAT'ing */
/* Filter rule value/mask pairs. */
struct ch_filter_tuple val;
@@ -111,6 +128,23 @@ enum {
FILTER_SWITCH
};
+enum {
+ VLAN_REMOVE = 1,
+ VLAN_INSERT,
+ VLAN_REWRITE
+};
+
+enum {
+ NAT_MODE_NONE = 0, /* No NAT performed */
+ NAT_MODE_DIP, /* NAT on Dst IP */
+ NAT_MODE_DIP_DP, /* NAT on Dst IP, Dst Port */
+ NAT_MODE_DIP_DP_SIP, /* NAT on Dst IP, Dst Port and Src IP */
+ NAT_MODE_DIP_DP_SP, /* NAT on Dst IP, Dst Port and Src Port */
+ NAT_MODE_SIP_SP, /* NAT on Src IP and Src Port */
+ NAT_MODE_DIP_SIP_SP, /* NAT on Dst IP, Src IP and Src Port */
+ NAT_MODE_ALL /* NAT on entire 4-tuple */
+};
+
enum filter_type {
FILTER_TYPE_IPV4 = 0,
FILTER_TYPE_IPV6,
@@ -145,6 +179,7 @@ struct filter_entry {
u32 pending:1; /* filter action is pending FW reply */
struct filter_ctx *ctx; /* caller's completion hook */
struct clip_entry *clipt; /* CLIP Table entry for IPv6 */
+ struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
struct rte_eth_dev *dev; /* Port's rte eth device */
void *private; /* For use by apps using filter_entry */
diff --git a/drivers/net/cxgbe/cxgbe_flow.c b/drivers/net/cxgbe/cxgbe_flow.c
index 01c945f1..54ec7e59 100644
--- a/drivers/net/cxgbe/cxgbe_flow.c
+++ b/drivers/net/cxgbe/cxgbe_flow.c
@@ -95,6 +95,8 @@ cxgbe_fill_filter_region(struct adapter *adap,
ntuple_mask |= (u64)fs->mask.ethtype << tp->ethertype_shift;
if (tp->port_shift >= 0)
ntuple_mask |= (u64)fs->mask.iport << tp->port_shift;
+ if (tp->macmatch_shift >= 0)
+ ntuple_mask |= (u64)fs->mask.macidx << tp->macmatch_shift;
if (ntuple_mask != hash_filter_mask)
return;
@@ -103,6 +105,46 @@ cxgbe_fill_filter_region(struct adapter *adap,
}
static int
+ch_rte_parsetype_eth(const void *dmask, const struct rte_flow_item *item,
+ struct ch_filter_specification *fs,
+ struct rte_flow_error *e)
+{
+ const struct rte_flow_item_eth *spec = item->spec;
+ const struct rte_flow_item_eth *umask = item->mask;
+ const struct rte_flow_item_eth *mask;
+
+ /* If user has not given any mask, then use chelsio supported mask. */
+ mask = umask ? umask : (const struct rte_flow_item_eth *)dmask;
+
+ /* we don't support SRC_MAC filtering*/
+ if (!is_zero_ether_addr(&mask->src))
+ return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "src mac filtering not supported");
+
+ if (!is_zero_ether_addr(&mask->dst)) {
+ const u8 *addr = (const u8 *)&spec->dst.addr_bytes[0];
+ const u8 *m = (const u8 *)&mask->dst.addr_bytes[0];
+ struct rte_flow *flow = (struct rte_flow *)fs->private;
+ struct port_info *pi = (struct port_info *)
+ (flow->dev->data->dev_private);
+ int idx;
+
+ idx = cxgbe_mpstcam_alloc(pi, addr, m);
+ if (idx <= 0)
+ return rte_flow_error_set(e, idx,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ NULL, "unable to allocate mac"
+ " entry in h/w");
+ CXGBE_FILL_FS(idx, 0x1ff, macidx);
+ }
+
+ CXGBE_FILL_FS(be16_to_cpu(spec->type),
+ be16_to_cpu(mask->type), ethtype);
+ return 0;
+}
+
+static int
ch_rte_parsetype_port(const void *dmask, const struct rte_flow_item *item,
struct ch_filter_specification *fs,
struct rte_flow_error *e)
@@ -327,17 +369,199 @@ static int cxgbe_get_fidx(struct rte_flow *flow, unsigned int *fidx)
}
static int
+cxgbe_get_flow_item_index(const struct rte_flow_item items[], u32 type)
+{
+ const struct rte_flow_item *i;
+ int j, index = -ENOENT;
+
+ for (i = items, j = 0; i->type != RTE_FLOW_ITEM_TYPE_END; i++, j++) {
+ if (i->type == type) {
+ index = j;
+ break;
+ }
+ }
+
+ return index;
+}
+
+static int
+ch_rte_parse_nat(uint8_t nmode, struct ch_filter_specification *fs)
+{
+ /* nmode:
+ * BIT_0 = [src_ip], BIT_1 = [dst_ip]
+ * BIT_2 = [src_port], BIT_3 = [dst_port]
+ *
+ * Only below cases are supported as per our spec.
+ */
+ switch (nmode) {
+ case 0: /* 0000b */
+ fs->nat_mode = NAT_MODE_NONE;
+ break;
+ case 2: /* 0010b */
+ fs->nat_mode = NAT_MODE_DIP;
+ break;
+ case 5: /* 0101b */
+ fs->nat_mode = NAT_MODE_SIP_SP;
+ break;
+ case 7: /* 0111b */
+ fs->nat_mode = NAT_MODE_DIP_SIP_SP;
+ break;
+ case 10: /* 1010b */
+ fs->nat_mode = NAT_MODE_DIP_DP;
+ break;
+ case 11: /* 1011b */
+ fs->nat_mode = NAT_MODE_DIP_DP_SIP;
+ break;
+ case 14: /* 1110b */
+ fs->nat_mode = NAT_MODE_DIP_DP_SP;
+ break;
+ case 15: /* 1111b */
+ fs->nat_mode = NAT_MODE_ALL;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int
ch_rte_parse_atype_switch(const struct rte_flow_action *a,
+ const struct rte_flow_item items[],
+ uint8_t *nmode,
struct ch_filter_specification *fs,
struct rte_flow_error *e)
{
+ const struct rte_flow_action_of_set_vlan_vid *vlanid;
+ const struct rte_flow_action_of_push_vlan *pushvlan;
+ const struct rte_flow_action_set_ipv4 *ipv4;
+ const struct rte_flow_action_set_ipv6 *ipv6;
+ const struct rte_flow_action_set_tp *tp_port;
const struct rte_flow_action_phy_port *port;
+ int item_index;
switch (a->type) {
+ case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
+ vlanid = (const struct rte_flow_action_of_set_vlan_vid *)
+ a->conf;
+ fs->newvlan = VLAN_REWRITE;
+ fs->vlan = vlanid->vlan_vid;
+ break;
+ case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
+ pushvlan = (const struct rte_flow_action_of_push_vlan *)
+ a->conf;
+ if (pushvlan->ethertype != ETHER_TYPE_VLAN)
+ return rte_flow_error_set(e, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, a,
+ "only ethertype 0x8100 "
+ "supported for push vlan.");
+ fs->newvlan = VLAN_INSERT;
+ break;
+ case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
+ fs->newvlan = VLAN_REMOVE;
+ break;
case RTE_FLOW_ACTION_TYPE_PHY_PORT:
port = (const struct rte_flow_action_phy_port *)a->conf;
fs->eport = port->index;
break;
+ case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
+ item_index = cxgbe_get_flow_item_index(items,
+ RTE_FLOW_ITEM_TYPE_IPV4);
+ if (item_index < 0)
+ return rte_flow_error_set(e, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, a,
+ "No RTE_FLOW_ITEM_TYPE_IPV4 "
+ "found.");
+
+ ipv4 = (const struct rte_flow_action_set_ipv4 *)a->conf;
+ memcpy(fs->nat_fip, &ipv4->ipv4_addr, sizeof(ipv4->ipv4_addr));
+ *nmode |= 1 << 0;
+ break;
+ case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
+ item_index = cxgbe_get_flow_item_index(items,
+ RTE_FLOW_ITEM_TYPE_IPV4);
+ if (item_index < 0)
+ return rte_flow_error_set(e, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, a,
+ "No RTE_FLOW_ITEM_TYPE_IPV4 "
+ "found.");
+
+ ipv4 = (const struct rte_flow_action_set_ipv4 *)a->conf;
+ memcpy(fs->nat_lip, &ipv4->ipv4_addr, sizeof(ipv4->ipv4_addr));
+ *nmode |= 1 << 1;
+ break;
+ case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
+ item_index = cxgbe_get_flow_item_index(items,
+ RTE_FLOW_ITEM_TYPE_IPV6);
+ if (item_index < 0)
+ return rte_flow_error_set(e, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, a,
+ "No RTE_FLOW_ITEM_TYPE_IPV6 "
+ "found.");
+
+ ipv6 = (const struct rte_flow_action_set_ipv6 *)a->conf;
+ memcpy(fs->nat_fip, ipv6->ipv6_addr, sizeof(ipv6->ipv6_addr));
+ *nmode |= 1 << 0;
+ break;
+ case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
+ item_index = cxgbe_get_flow_item_index(items,
+ RTE_FLOW_ITEM_TYPE_IPV6);
+ if (item_index < 0)
+ return rte_flow_error_set(e, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, a,
+ "No RTE_FLOW_ITEM_TYPE_IPV6 "
+ "found.");
+
+ ipv6 = (const struct rte_flow_action_set_ipv6 *)a->conf;
+ memcpy(fs->nat_lip, ipv6->ipv6_addr, sizeof(ipv6->ipv6_addr));
+ *nmode |= 1 << 1;
+ break;
+ case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
+ item_index = cxgbe_get_flow_item_index(items,
+ RTE_FLOW_ITEM_TYPE_TCP);
+ if (item_index < 0) {
+ item_index =
+ cxgbe_get_flow_item_index(items,
+ RTE_FLOW_ITEM_TYPE_UDP);
+ if (item_index < 0)
+ return rte_flow_error_set(e, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, a,
+ "No RTE_FLOW_ITEM_TYPE_TCP or "
+ "RTE_FLOW_ITEM_TYPE_UDP found");
+ }
+
+ tp_port = (const struct rte_flow_action_set_tp *)a->conf;
+ fs->nat_fport = be16_to_cpu(tp_port->port);
+ *nmode |= 1 << 2;
+ break;
+ case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
+ item_index = cxgbe_get_flow_item_index(items,
+ RTE_FLOW_ITEM_TYPE_TCP);
+ if (item_index < 0) {
+ item_index =
+ cxgbe_get_flow_item_index(items,
+ RTE_FLOW_ITEM_TYPE_UDP);
+ if (item_index < 0)
+ return rte_flow_error_set(e, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, a,
+ "No RTE_FLOW_ITEM_TYPE_TCP or "
+ "RTE_FLOW_ITEM_TYPE_UDP found");
+ }
+
+ tp_port = (const struct rte_flow_action_set_tp *)a->conf;
+ fs->nat_lport = be16_to_cpu(tp_port->port);
+ *nmode |= 1 << 3;
+ break;
+ case RTE_FLOW_ACTION_TYPE_MAC_SWAP:
+ item_index = cxgbe_get_flow_item_index(items,
+ RTE_FLOW_ITEM_TYPE_ETH);
+ if (item_index < 0)
+ return rte_flow_error_set(e, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, a,
+ "No RTE_FLOW_ITEM_TYPE_ETH "
+ "found");
+ fs->swapmac = 1;
+ break;
default:
/* We are not supposed to come here */
return rte_flow_error_set(e, EINVAL,
@@ -350,10 +574,12 @@ ch_rte_parse_atype_switch(const struct rte_flow_action *a,
static int
cxgbe_rtef_parse_actions(struct rte_flow *flow,
+ const struct rte_flow_item items[],
const struct rte_flow_action action[],
struct rte_flow_error *e)
{
struct ch_filter_specification *fs = &flow->fs;
+ uint8_t nmode = 0, nat_ipv4 = 0, nat_ipv6 = 0;
const struct rte_flow_action_queue *q;
const struct rte_flow_action *a;
char abit = 0;
@@ -391,7 +617,22 @@ cxgbe_rtef_parse_actions(struct rte_flow *flow,
case RTE_FLOW_ACTION_TYPE_COUNT:
fs->hitcnts = 1;
break;
+ case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
+ case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
+ case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
case RTE_FLOW_ACTION_TYPE_PHY_PORT:
+ case RTE_FLOW_ACTION_TYPE_MAC_SWAP:
+ case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
+ case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
+ nat_ipv4++;
+ goto action_switch;
+ case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
+ case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
+ nat_ipv6++;
+ goto action_switch;
+ case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
+ case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
+action_switch:
/* We allow multiple switch actions, but switch is
* not compatible with either queue or drop
*/
@@ -399,7 +640,14 @@ cxgbe_rtef_parse_actions(struct rte_flow *flow,
return rte_flow_error_set(e, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, a,
"overlapping action specified");
- ret = ch_rte_parse_atype_switch(a, fs, e);
+ if (nat_ipv4 && nat_ipv6)
+ return rte_flow_error_set(e, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, a,
+ "Can't have one address ipv4 and the"
+ " other ipv6");
+
+ ret = ch_rte_parse_atype_switch(a, items, &nmode, fs,
+ e);
if (ret)
return ret;
fs->action = FILTER_SWITCH;
@@ -412,11 +660,24 @@ cxgbe_rtef_parse_actions(struct rte_flow *flow,
}
}
+ if (ch_rte_parse_nat(nmode, fs))
+ return rte_flow_error_set(e, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, a,
+ "invalid settings for swich action");
return 0;
}
-struct chrte_fparse parseitem[] = {
- [RTE_FLOW_ITEM_TYPE_PHY_PORT] = {
+static struct chrte_fparse parseitem[] = {
+ [RTE_FLOW_ITEM_TYPE_ETH] = {
+ .fptr = ch_rte_parsetype_eth,
+ .dmask = &(const struct rte_flow_item_eth){
+ .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+ .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+ .type = 0xffff,
+ }
+ },
+
+ [RTE_FLOW_ITEM_TYPE_PHY_PORT] = {
.fptr = ch_rte_parsetype_port,
.dmask = &(const struct rte_flow_item_phy_port){
.index = 0x7,
@@ -454,10 +715,10 @@ cxgbe_rtef_parse_items(struct rte_flow *flow,
char repeat[ARRAY_SIZE(parseitem)] = {0};
for (i = items; i->type != RTE_FLOW_ITEM_TYPE_END; i++) {
- struct chrte_fparse *idx = &flow->item_parser[i->type];
+ struct chrte_fparse *idx;
int ret;
- if (i->type > ARRAY_SIZE(parseitem))
+ if (i->type >= ARRAY_SIZE(parseitem))
return rte_flow_error_set(e, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ITEM,
i, "Item not supported");
@@ -478,6 +739,7 @@ cxgbe_rtef_parse_items(struct rte_flow *flow,
if (ret)
return ret;
+ idx = &flow->item_parser[i->type];
if (!idx || !idx->fptr) {
return rte_flow_error_set(e, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ITEM, i,
@@ -503,7 +765,6 @@ cxgbe_flow_parse(struct rte_flow *flow,
struct rte_flow_error *e)
{
int ret;
-
/* parse user request into ch_filter_specification */
ret = cxgbe_rtef_parse_attr(flow, attr, e);
if (ret)
@@ -511,7 +772,7 @@ cxgbe_flow_parse(struct rte_flow *flow,
ret = cxgbe_rtef_parse_items(flow, item, e);
if (ret)
return ret;
- return cxgbe_rtef_parse_actions(flow, action, e);
+ return cxgbe_rtef_parse_actions(flow, item, action, e);
}
static int __cxgbe_flow_create(struct rte_eth_dev *dev, struct rte_flow *flow)
@@ -582,6 +843,7 @@ cxgbe_flow_create(struct rte_eth_dev *dev,
flow->item_parser = parseitem;
flow->dev = dev;
+ flow->fs.private = (void *)flow;
if (cxgbe_flow_parse(flow, attr, item, action, e)) {
t4_os_free(flow);
@@ -636,6 +898,17 @@ static int __cxgbe_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
return ctx.result;
}
+ fs = &flow->fs;
+ if (fs->mask.macidx) {
+ struct port_info *pi = (struct port_info *)
+ (dev->data->dev_private);
+ int ret;
+
+ ret = cxgbe_mpstcam_remove(pi, fs->val.macidx);
+ if (!ret)
+ return ret;
+ }
+
return 0;
}
diff --git a/drivers/net/cxgbe/cxgbe_flow.h b/drivers/net/cxgbe/cxgbe_flow.h
index 0f750474..718bf3d0 100644
--- a/drivers/net/cxgbe/cxgbe_flow.h
+++ b/drivers/net/cxgbe/cxgbe_flow.h
@@ -7,6 +7,7 @@
#include <rte_flow_driver.h>
#include "cxgbe_filter.h"
+#include "mps_tcam.h"
#include "cxgbe.h"
#define CXGBE_FLOW_POLL_US 10
diff --git a/drivers/net/cxgbe/cxgbe_main.c b/drivers/net/cxgbe/cxgbe_main.c
index c3938e8d..88dc851f 100644
--- a/drivers/net/cxgbe/cxgbe_main.c
+++ b/drivers/net/cxgbe/cxgbe_main.c
@@ -38,6 +38,8 @@
#include "t4_msg.h"
#include "cxgbe.h"
#include "clip_tbl.h"
+#include "l2t.h"
+#include "mps_tcam.h"
/**
* Allocate a chunk of memory. The allocated memory is cleared.
@@ -99,6 +101,10 @@ static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
const struct cpl_act_open_rpl *p = (const void *)rsp;
hash_filter_rpl(q->adapter, p);
+ } else if (opcode == CPL_L2T_WRITE_RPL) {
+ const struct cpl_l2t_write_rpl *p = (const void *)rsp;
+
+ do_l2t_write_rpl(q->adapter, p);
} else {
dev_err(adapter, "unexpected CPL %#x on FW event queue\n",
opcode);
@@ -1135,13 +1141,17 @@ static int adap_init0(struct adapter *adap)
V_FW_PARAMS_PARAM_Y(0) | \
V_FW_PARAMS_PARAM_Z(0))
- params[0] = FW_PARAM_PFVF(FILTER_START);
- params[1] = FW_PARAM_PFVF(FILTER_END);
- ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
+ params[0] = FW_PARAM_PFVF(L2T_START);
+ params[1] = FW_PARAM_PFVF(L2T_END);
+ params[2] = FW_PARAM_PFVF(FILTER_START);
+ params[3] = FW_PARAM_PFVF(FILTER_END);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 4, params, val);
if (ret < 0)
goto bye;
- adap->tids.ftid_base = val[0];
- adap->tids.nftids = val[1] - val[0] + 1;
+ adap->l2t_start = val[0];
+ adap->l2t_end = val[1];
+ adap->tids.ftid_base = val[2];
+ adap->tids.nftids = val[3] - val[2] + 1;
params[0] = FW_PARAM_PFVF(CLIP_START);
params[1] = FW_PARAM_PFVF(CLIP_END);
@@ -1170,6 +1180,16 @@ static int adap_init0(struct adapter *adap)
goto bye;
}
+ /* See if FW supports FW_FILTER2 work request */
+ if (is_t4(adap->params.chip)) {
+ adap->params.filter2_wr_support = 0;
+ } else {
+ params[0] = FW_PARAM_DEV(FILTER2_WR);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 1, params, val);
+ adap->params.filter2_wr_support = (ret == 0 && val[0] != 0);
+ }
+
/* query tid-related parameters */
params[0] = FW_PARAM_DEV(NTID);
ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1,
@@ -1332,10 +1352,8 @@ int link_start(struct port_info *pi)
ret = t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu, -1, -1,
-1, 1, true);
if (ret == 0) {
- ret = t4_change_mac(adapter, adapter->mbox, pi->viid,
- pi->xact_addr_filt,
- (u8 *)&pi->eth_dev->data->mac_addrs[0],
- true, true);
+ ret = cxgbe_mpstcam_modify(pi, (int)pi->xact_addr_filt,
+ (u8 *)&pi->eth_dev->data->mac_addrs[0]);
if (ret >= 0) {
pi->xact_addr_filt = ret;
ret = 0;
@@ -1679,10 +1697,12 @@ void cxgbe_close(struct adapter *adapter)
int i;
if (adapter->flags & FULL_INIT_DONE) {
- if (is_pf4(adapter))
- t4_intr_disable(adapter);
tid_free(&adapter->tids);
+ t4_cleanup_mpstcam(adapter);
t4_cleanup_clip_tbl(adapter);
+ t4_cleanup_l2t(adapter);
+ if (is_pf4(adapter))
+ t4_intr_disable(adapter);
t4_sge_tx_monitor_stop(adapter);
t4_free_sge_resources(adapter);
for_each_port(adapter, i) {
@@ -1690,12 +1710,7 @@ void cxgbe_close(struct adapter *adapter)
if (pi->viid != 0)
t4_free_vi(adapter, adapter->mbox,
adapter->pf, 0, pi->viid);
- rte_free(pi->eth_dev->data->mac_addrs);
- /* Skip first port since it'll be freed by DPDK stack */
- if (i) {
- rte_free(pi->eth_dev->data->dev_private);
- rte_eth_dev_release_port(pi->eth_dev);
- }
+ rte_eth_dev_release_port(pi->eth_dev);
}
adapter->flags &= ~FULL_INIT_DONE;
}
@@ -1855,12 +1870,23 @@ allocate_mac:
dev_warn(adapter, "could not allocate CLIP. Continuing\n");
}
+ adapter->l2t = t4_init_l2t(adapter->l2t_start, adapter->l2t_end);
+ if (!adapter->l2t) {
+ /* We tolerate a lack of L2T, giving up some functionality */
+ dev_warn(adapter, "could not allocate L2T. Continuing\n");
+ }
+
if (tid_init(&adapter->tids) < 0) {
/* Disable filtering support */
dev_warn(adapter, "could not allocate TID table, "
"filter support disabled. Continuing\n");
}
+ adapter->mpstcam = t4_init_mpstcam(adapter);
+ if (!adapter->mpstcam)
+ dev_warn(adapter, "could not allocate mps tcam table."
+ " Continuing\n");
+
if (is_hashfilter(adapter)) {
if (t4_read_reg(adapter, A_LE_DB_CONFIG) & F_HASHEN) {
u32 hash_base, hash_reg;
@@ -1887,14 +1913,7 @@ out_free:
if (pi->viid != 0)
t4_free_vi(adapter, adapter->mbox, adapter->pf,
0, pi->viid);
- /* Skip first port since it'll be de-allocated by DPDK */
- if (i == 0)
- continue;
- if (pi->eth_dev) {
- if (pi->eth_dev->data->dev_private)
- rte_free(pi->eth_dev->data->dev_private);
- rte_eth_dev_release_port(pi->eth_dev);
- }
+ rte_eth_dev_release_port(pi->eth_dev);
}
if (adapter->flags & FW_OK)
diff --git a/drivers/net/cxgbe/cxgbevf_main.c b/drivers/net/cxgbe/cxgbevf_main.c
index 4214d031..6223e125 100644
--- a/drivers/net/cxgbe/cxgbevf_main.c
+++ b/drivers/net/cxgbe/cxgbevf_main.c
@@ -282,14 +282,7 @@ out_free:
if (pi->viid != 0)
t4_free_vi(adapter, adapter->mbox, adapter->pf,
0, pi->viid);
- /* Skip first port since it'll be de-allocated by DPDK */
- if (i == 0)
- continue;
- if (pi->eth_dev) {
- if (pi->eth_dev->data->dev_private)
- rte_free(pi->eth_dev->data->dev_private);
- rte_eth_dev_release_port(pi->eth_dev);
- }
+ rte_eth_dev_release_port(pi->eth_dev);
}
return -err;
}
diff --git a/drivers/net/cxgbe/l2t.c b/drivers/net/cxgbe/l2t.c
new file mode 100644
index 00000000..814188fe
--- /dev/null
+++ b/drivers/net/cxgbe/l2t.c
@@ -0,0 +1,227 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+#include "common.h"
+#include "l2t.h"
+
+/**
+ * cxgbe_l2t_release - Release associated L2T entry
+ * @e: L2T entry to release
+ *
+ * Releases ref count and frees up an L2T entry from L2T table
+ */
+void cxgbe_l2t_release(struct l2t_entry *e)
+{
+ if (rte_atomic32_read(&e->refcnt) != 0)
+ rte_atomic32_dec(&e->refcnt);
+}
+
+/**
+ * Process a CPL_L2T_WRITE_RPL. Note that the TID in the reply is really
+ * the L2T index it refers to.
+ */
+void do_l2t_write_rpl(struct adapter *adap, const struct cpl_l2t_write_rpl *rpl)
+{
+ struct l2t_data *d = adap->l2t;
+ unsigned int tid = GET_TID(rpl);
+ unsigned int l2t_idx = tid % L2T_SIZE;
+
+ if (unlikely(rpl->status != CPL_ERR_NONE)) {
+ dev_err(adap,
+ "Unexpected L2T_WRITE_RPL status %u for entry %u\n",
+ rpl->status, l2t_idx);
+ return;
+ }
+
+ if (tid & F_SYNC_WR) {
+ struct l2t_entry *e = &d->l2tab[l2t_idx - d->l2t_start];
+
+ t4_os_lock(&e->lock);
+ if (e->state != L2T_STATE_SWITCHING)
+ e->state = L2T_STATE_VALID;
+ t4_os_unlock(&e->lock);
+ }
+}
+
+/**
+ * Write an L2T entry. Must be called with the entry locked.
+ * The write may be synchronous or asynchronous.
+ */
+static int write_l2e(struct rte_eth_dev *dev, struct l2t_entry *e, int sync,
+ bool loopback, bool arpmiss)
+{
+ struct adapter *adap = ethdev2adap(dev);
+ struct l2t_data *d = adap->l2t;
+ struct rte_mbuf *mbuf;
+ struct cpl_l2t_write_req *req;
+ struct sge_ctrl_txq *ctrlq;
+ unsigned int l2t_idx = e->idx + d->l2t_start;
+ unsigned int port_id = ethdev2pinfo(dev)->port_id;
+
+ ctrlq = &adap->sge.ctrlq[port_id];
+ mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool);
+ if (!mbuf)
+ return -ENOMEM;
+
+ mbuf->data_len = sizeof(*req);
+ mbuf->pkt_len = mbuf->data_len;
+
+ req = rte_pktmbuf_mtod(mbuf, struct cpl_l2t_write_req *);
+ INIT_TP_WR(req, 0);
+
+ OPCODE_TID(req) =
+ cpu_to_be32(MK_OPCODE_TID(CPL_L2T_WRITE_REQ,
+ l2t_idx | V_SYNC_WR(sync) |
+ V_TID_QID(adap->sge.fw_evtq.abs_id)));
+ req->params = cpu_to_be16(V_L2T_W_PORT(e->lport) |
+ V_L2T_W_LPBK(loopback) |
+ V_L2T_W_ARPMISS(arpmiss) |
+ V_L2T_W_NOREPLY(!sync));
+ req->l2t_idx = cpu_to_be16(l2t_idx);
+ req->vlan = cpu_to_be16(e->vlan);
+ rte_memcpy(req->dst_mac, e->dmac, ETHER_ADDR_LEN);
+
+ if (loopback)
+ memset(req->dst_mac, 0, ETHER_ADDR_LEN);
+
+ t4_mgmt_tx(ctrlq, mbuf);
+
+ if (sync && e->state != L2T_STATE_SWITCHING)
+ e->state = L2T_STATE_SYNC_WRITE;
+
+ return 0;
+}
+
+/**
+ * find_or_alloc_l2e - Find/Allocate a free L2T entry
+ * @d: L2T table
+ * @vlan: VLAN id to compare/add
+ * @port: port id to compare/add
+ * @dmac: Destination MAC address to compare/add
+ * Returns pointer to the L2T entry found/created
+ *
+ * Finds/Allocates an L2T entry to be used by switching rule of a filter.
+ */
+static struct l2t_entry *find_or_alloc_l2e(struct l2t_data *d, u16 vlan,
+ u8 port, u8 *dmac)
+{
+ struct l2t_entry *end, *e;
+ struct l2t_entry *first_free = NULL;
+
+ for (e = &d->l2tab[0], end = &d->l2tab[d->l2t_size]; e != end; ++e) {
+ if (rte_atomic32_read(&e->refcnt) == 0) {
+ if (!first_free)
+ first_free = e;
+ } else {
+ if (e->state == L2T_STATE_SWITCHING) {
+ if ((!memcmp(e->dmac, dmac, ETHER_ADDR_LEN)) &&
+ e->vlan == vlan && e->lport == port)
+ goto exists;
+ }
+ }
+ }
+
+ if (first_free) {
+ e = first_free;
+ goto found;
+ }
+
+ return NULL;
+
+found:
+ e->state = L2T_STATE_UNUSED;
+
+exists:
+ return e;
+}
+
+static struct l2t_entry *t4_l2t_alloc_switching(struct rte_eth_dev *dev,
+ u16 vlan, u8 port,
+ u8 *eth_addr)
+{
+ struct adapter *adap = ethdev2adap(dev);
+ struct l2t_data *d = adap->l2t;
+ struct l2t_entry *e;
+ int ret = 0;
+
+ t4_os_write_lock(&d->lock);
+ e = find_or_alloc_l2e(d, vlan, port, eth_addr);
+ if (e) {
+ t4_os_lock(&e->lock);
+ if (!rte_atomic32_read(&e->refcnt)) {
+ e->state = L2T_STATE_SWITCHING;
+ e->vlan = vlan;
+ e->lport = port;
+ rte_memcpy(e->dmac, eth_addr, ETHER_ADDR_LEN);
+ rte_atomic32_set(&e->refcnt, 1);
+ ret = write_l2e(dev, e, 0, !L2T_LPBK, !L2T_ARPMISS);
+ if (ret < 0)
+ dev_debug(adap, "Failed to write L2T entry: %d",
+ ret);
+ } else {
+ rte_atomic32_inc(&e->refcnt);
+ }
+ t4_os_unlock(&e->lock);
+ }
+ t4_os_write_unlock(&d->lock);
+
+ return ret ? NULL : e;
+}
+
+/**
+ * cxgbe_l2t_alloc_switching - Allocate a L2T entry for switching rule
+ * @dev: rte_eth_dev pointer
+ * @vlan: VLAN Id
+ * @port: Associated port
+ * @dmac: Destination MAC address to add to L2T
+ * Returns pointer to the allocated l2t entry
+ *
+ * Allocates a L2T entry for use by switching rule of a filter
+ */
+struct l2t_entry *cxgbe_l2t_alloc_switching(struct rte_eth_dev *dev, u16 vlan,
+ u8 port, u8 *dmac)
+{
+ return t4_l2t_alloc_switching(dev, vlan, port, dmac);
+}
+
+/**
+ * Initialize L2 Table
+ */
+struct l2t_data *t4_init_l2t(unsigned int l2t_start, unsigned int l2t_end)
+{
+ unsigned int l2t_size;
+ unsigned int i;
+ struct l2t_data *d;
+
+ if (l2t_start >= l2t_end || l2t_end >= L2T_SIZE)
+ return NULL;
+ l2t_size = l2t_end - l2t_start + 1;
+
+ d = t4_os_alloc(sizeof(*d) + l2t_size * sizeof(struct l2t_entry));
+ if (!d)
+ return NULL;
+
+ d->l2t_start = l2t_start;
+ d->l2t_size = l2t_size;
+
+ t4_os_rwlock_init(&d->lock);
+
+ for (i = 0; i < d->l2t_size; ++i) {
+ d->l2tab[i].idx = i;
+ d->l2tab[i].state = L2T_STATE_UNUSED;
+ t4_os_lock_init(&d->l2tab[i].lock);
+ rte_atomic32_set(&d->l2tab[i].refcnt, 0);
+ }
+
+ return d;
+}
+
+/**
+ * Cleanup L2 Table
+ */
+void t4_cleanup_l2t(struct adapter *adap)
+{
+ if (adap->l2t)
+ t4_os_free(adap->l2t);
+}
diff --git a/drivers/net/cxgbe/l2t.h b/drivers/net/cxgbe/l2t.h
new file mode 100644
index 00000000..22a34e38
--- /dev/null
+++ b/drivers/net/cxgbe/l2t.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+#ifndef _CXGBE_L2T_H_
+#define _CXGBE_L2T_H_
+
+#include "t4_msg.h"
+
+enum {
+ L2T_SIZE = 4096 /* # of L2T entries */
+};
+
+enum {
+ L2T_STATE_VALID, /* entry is up to date */
+ L2T_STATE_SYNC_WRITE, /* synchronous write of entry underway */
+
+ /* when state is one of the below the entry is not hashed */
+ L2T_STATE_SWITCHING, /* entry is being used by a switching filter */
+ L2T_STATE_UNUSED /* entry not in use */
+};
+
+/*
+ * State for the corresponding entry of the HW L2 table.
+ */
+struct l2t_entry {
+ u16 state; /* entry state */
+ u16 idx; /* entry index within in-memory table */
+ u16 vlan; /* VLAN TCI (id: bits 0-11, prio: 13-15 */
+ u8 lport; /* destination port */
+ u8 dmac[ETHER_ADDR_LEN]; /* destination MAC address */
+ rte_spinlock_t lock; /* entry lock */
+ rte_atomic32_t refcnt; /* entry reference count */
+};
+
+struct l2t_data {
+ unsigned int l2t_start; /* start index of our piece of the L2T */
+ unsigned int l2t_size; /* number of entries in l2tab */
+ rte_rwlock_t lock; /* table rw lock */
+ struct l2t_entry l2tab[0]; /* MUST BE LAST */
+};
+
+#define L2T_LPBK true
+#define L2T_ARPMISS true
+
+/* identifies sync vs async L2T_WRITE_REQs */
+#define S_SYNC_WR 12
+#define V_SYNC_WR(x) ((x) << S_SYNC_WR)
+#define F_SYNC_WR V_SYNC_WR(1)
+
+struct l2t_data *t4_init_l2t(unsigned int l2t_start, unsigned int l2t_end);
+void t4_cleanup_l2t(struct adapter *adap);
+struct l2t_entry *cxgbe_l2t_alloc_switching(struct rte_eth_dev *dev, u16 vlan,
+ u8 port, u8 *dmac);
+void cxgbe_l2t_release(struct l2t_entry *e);
+void do_l2t_write_rpl(struct adapter *p, const struct cpl_l2t_write_rpl *rpl);
+#endif /* _CXGBE_L2T_H_ */
diff --git a/drivers/net/cxgbe/meson.build b/drivers/net/cxgbe/meson.build
index 7c69a34b..c51af26e 100644
--- a/drivers/net/cxgbe/meson.build
+++ b/drivers/net/cxgbe/meson.build
@@ -9,6 +9,8 @@ sources = files('cxgbe_ethdev.c',
'cxgbe_filter.c',
'cxgbe_flow.c',
'clip_tbl.c',
+ 'mps_tcam.c',
+ 'l2t.c',
'base/t4_hw.c',
'base/t4vf_hw.c')
includes += include_directories('base')
diff --git a/drivers/net/cxgbe/mps_tcam.c b/drivers/net/cxgbe/mps_tcam.c
new file mode 100644
index 00000000..02ec69a9
--- /dev/null
+++ b/drivers/net/cxgbe/mps_tcam.c
@@ -0,0 +1,243 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+
+#include "mps_tcam.h"
+
+static inline bool
+match_entry(struct mps_tcam_entry *entry, const u8 *eth_addr, const u8 *mask)
+{
+ if (!memcmp(eth_addr, entry->eth_addr, ETHER_ADDR_LEN) &&
+ !memcmp(mask, entry->mask, ETHER_ADDR_LEN))
+ return true;
+ return false;
+}
+
+static int cxgbe_update_free_idx(struct mpstcam_table *t)
+{
+ struct mps_tcam_entry *entry = t->entry;
+ u16 i, next = t->free_idx + 1;
+
+ if (entry[t->free_idx].state == MPS_ENTRY_UNUSED)
+ /* You are already pointing to a free entry !! */
+ return 0;
+
+ /* loop, till we don't rollback to same index where we started */
+ for (i = next; i != t->free_idx; i++) {
+ if (i == t->size)
+ /* rollback and search free entry from start */
+ i = 0;
+
+ if (entry[i].state == MPS_ENTRY_UNUSED) {
+ t->free_idx = i;
+ return 0;
+ }
+ }
+
+ return -1; /* table is full */
+}
+
+static struct mps_tcam_entry *
+cxgbe_mpstcam_lookup(struct mpstcam_table *t, const u8 *eth_addr,
+ const u8 *mask)
+{
+ struct mps_tcam_entry *entry = t->entry;
+ int i;
+
+ if (!entry)
+ return NULL;
+
+ for (i = 0; i < t->size; i++) {
+ if (entry[i].state == MPS_ENTRY_UNUSED)
+ continue; /* entry is not being used */
+ if (match_entry(&entry[i], eth_addr, mask))
+ return &entry[i];
+ }
+
+ return NULL;
+}
+
+int cxgbe_mpstcam_alloc(struct port_info *pi, const u8 *eth_addr,
+ const u8 *mask)
+{
+ struct adapter *adap = pi->adapter;
+ struct mpstcam_table *mpstcam = adap->mpstcam;
+ struct mps_tcam_entry *entry;
+ int ret;
+
+ if (!adap->mpstcam) {
+ dev_err(adap, "mpstcam table is not available\n");
+ return -EOPNOTSUPP;
+ }
+
+ /* If entry already present, return it. */
+ t4_os_write_lock(&mpstcam->lock);
+ entry = cxgbe_mpstcam_lookup(adap->mpstcam, eth_addr, mask);
+ if (entry) {
+ rte_atomic32_add(&entry->refcnt, 1);
+ t4_os_write_unlock(&mpstcam->lock);
+ return entry->idx;
+ }
+
+ if (mpstcam->full) {
+ t4_os_write_unlock(&mpstcam->lock);
+ dev_err(adap, "mps-tcam table is full\n");
+ return -ENOMEM;
+ }
+
+ ret = t4_alloc_raw_mac_filt(adap, pi->viid, eth_addr, mask,
+ mpstcam->free_idx, 0, pi->port_id, false);
+ if (ret <= 0) {
+ t4_os_write_unlock(&mpstcam->lock);
+ return ret;
+ }
+
+ /* Fill in the new values */
+ entry = &mpstcam->entry[ret];
+ memcpy(entry->eth_addr, eth_addr, ETHER_ADDR_LEN);
+ memcpy(entry->mask, mask, ETHER_ADDR_LEN);
+ rte_atomic32_set(&entry->refcnt, 1);
+ entry->state = MPS_ENTRY_USED;
+
+ if (cxgbe_update_free_idx(mpstcam))
+ mpstcam->full = true;
+
+ t4_os_write_unlock(&mpstcam->lock);
+ return ret;
+}
+
+int cxgbe_mpstcam_modify(struct port_info *pi, int idx, const u8 *addr)
+{
+ struct adapter *adap = pi->adapter;
+ struct mpstcam_table *mpstcam = adap->mpstcam;
+ struct mps_tcam_entry *entry;
+
+ if (!mpstcam)
+ return -EOPNOTSUPP;
+ t4_os_write_lock(&mpstcam->lock);
+ if (idx != -1 && idx >= mpstcam->size) {
+ t4_os_write_unlock(&mpstcam->lock);
+ return -EINVAL;
+ }
+ if (idx >= 0) {
+ entry = &mpstcam->entry[idx];
+ /* user wants to modify an existing entry.
+ * verify if entry exists
+ */
+ if (entry->state != MPS_ENTRY_USED) {
+ t4_os_write_unlock(&mpstcam->lock);
+ return -EINVAL;
+ }
+ }
+
+ idx = t4_change_mac(adap, adap->mbox, pi->viid, idx, addr, true, true);
+ if (idx < 0) {
+ t4_os_write_unlock(&mpstcam->lock);
+ return idx;
+ }
+
+ /* idx can now be different from what user provided */
+ entry = &mpstcam->entry[idx];
+ memcpy(entry->eth_addr, addr, ETHER_ADDR_LEN);
+ /* NOTE: we have considered the case that idx returned by t4_change_mac
+ * will be different from the user provided value only if user
+ * provided value is -1
+ */
+ if (entry->state == MPS_ENTRY_UNUSED) {
+ rte_atomic32_set(&entry->refcnt, 1);
+ entry->state = MPS_ENTRY_USED;
+ }
+
+ if (cxgbe_update_free_idx(mpstcam))
+ mpstcam->full = true;
+
+ t4_os_write_unlock(&mpstcam->lock);
+ return idx;
+}
+
+/**
+ * hold appropriate locks while calling this.
+ */
+static inline void reset_mpstcam_entry(struct mps_tcam_entry *entry)
+{
+ memset(entry->eth_addr, 0, ETHER_ADDR_LEN);
+ memset(entry->mask, 0, ETHER_ADDR_LEN);
+ rte_atomic32_clear(&entry->refcnt);
+ entry->state = MPS_ENTRY_UNUSED;
+}
+
+/**
+ * ret < 0: fatal error
+ * ret = 0: entry removed in h/w
+ * ret > 0: updated refcount.
+ */
+int cxgbe_mpstcam_remove(struct port_info *pi, u16 idx)
+{
+ struct adapter *adap = pi->adapter;
+ struct mpstcam_table *t = adap->mpstcam;
+ struct mps_tcam_entry *entry;
+ int ret;
+
+ if (!t)
+ return -EOPNOTSUPP;
+ t4_os_write_lock(&t->lock);
+ entry = &t->entry[idx];
+ if (entry->state == MPS_ENTRY_UNUSED) {
+ t4_os_write_unlock(&t->lock);
+ return -EINVAL;
+ }
+
+ if (rte_atomic32_read(&entry->refcnt) == 1)
+ ret = t4_free_raw_mac_filt(adap, pi->viid, entry->eth_addr,
+ entry->mask, idx, 1, pi->port_id,
+ false);
+ else
+ ret = rte_atomic32_sub_return(&entry->refcnt, 1);
+
+ if (ret == 0) {
+ reset_mpstcam_entry(entry);
+ t->full = false; /* We have atleast 1 free entry */
+ cxgbe_update_free_idx(t);
+ }
+
+ t4_os_write_unlock(&t->lock);
+ return ret;
+}
+
+struct mpstcam_table *t4_init_mpstcam(struct adapter *adap)
+{
+ struct mpstcam_table *t;
+ int i;
+ u16 size = adap->params.arch.mps_tcam_size;
+
+ t = t4_os_alloc(sizeof(*t) + size * sizeof(struct mps_tcam_entry));
+ if (!t)
+ return NULL;
+
+ t4_os_rwlock_init(&t->lock);
+ t->full = false;
+ t->size = size;
+
+ for (i = 0; i < size; i++) {
+ reset_mpstcam_entry(&t->entry[i]);
+ t->entry[i].mpstcam = t;
+ t->entry[i].idx = i;
+ }
+
+ /* first entry is used by chip. this is overwritten only
+ * in t4_cleanup_mpstcam()
+ */
+ t->entry[0].state = MPS_ENTRY_USED;
+ t->free_idx = 1;
+
+ return t;
+}
+
+void t4_cleanup_mpstcam(struct adapter *adap)
+{
+ if (adap->mpstcam) {
+ t4_os_free(adap->mpstcam->entry);
+ t4_os_free(adap->mpstcam);
+ }
+}
diff --git a/drivers/net/cxgbe/mps_tcam.h b/drivers/net/cxgbe/mps_tcam.h
new file mode 100644
index 00000000..c3d6fe0d
--- /dev/null
+++ b/drivers/net/cxgbe/mps_tcam.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+
+#ifndef _CXGBE_MPSTCAM_H_
+#define _CXGBE_MPSTCAM_H_
+
+#include "common.h"
+
+enum {
+ MPS_ENTRY_UNUSED, /* Keep this first so memset 0 renders
+ * the correct state. Other states can
+ * be added in future like MPS_ENTRY_BUSY
+ * to reduce contention while mboxing
+ * the request to f/w or to denote attributes
+ * for a specific entry
+ */
+ MPS_ENTRY_USED,
+};
+
+struct mps_tcam_entry {
+ u8 state;
+ u16 idx;
+
+ /* add data here which uniquely defines an entry */
+ u8 eth_addr[ETHER_ADDR_LEN];
+ u8 mask[ETHER_ADDR_LEN];
+
+ struct mpstcam_table *mpstcam; /* backptr */
+ rte_atomic32_t refcnt;
+};
+
+struct mpstcam_table {
+ u16 size;
+ rte_rwlock_t lock;
+ u16 free_idx; /* next free index */
+ bool full; /* since free index can be present
+ * anywhere in the table, size and
+ * free_idx cannot alone determine
+ * if the table is full
+ */
+ struct mps_tcam_entry entry[0];
+};
+
+struct mpstcam_table *t4_init_mpstcam(struct adapter *adap);
+void t4_cleanup_mpstcam(struct adapter *adap);
+int cxgbe_mpstcam_alloc(struct port_info *pi, const u8 *mac, const u8 *mask);
+int cxgbe_mpstcam_remove(struct port_info *pi, u16 idx);
+int cxgbe_mpstcam_modify(struct port_info *pi, int idx, const u8 *addr);
+
+#endif /* _CXGBE_MPSTCAM_H_ */
diff --git a/drivers/net/cxgbe/sge.c b/drivers/net/cxgbe/sge.c
index 4ea40d19..f9d2d48a 100644
--- a/drivers/net/cxgbe/sge.c
+++ b/drivers/net/cxgbe/sge.c
@@ -1873,10 +1873,9 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
/* Size needs to be multiple of 16, including status entry. */
iq->size = cxgbe_roundup(iq->size, 16);
- snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
- eth_dev->device->driver->name,
- fwevtq ? "fwq_ring" : "rx_ring",
- eth_dev->data->port_id, queue_id);
+ snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
+ eth_dev->data->port_id, queue_id,
+ fwevtq ? "fwq_ring" : "rx_ring");
snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
iq->desc = alloc_ring(iq->size, iq->iqe_len, 0, &iq->phys_addr, NULL, 0,
@@ -1938,10 +1937,9 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
fl->size = s->fl_starve_thres - 1 + 2 * 8;
fl->size = cxgbe_roundup(fl->size, 8);
- snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
- eth_dev->device->driver->name,
- fwevtq ? "fwq_ring" : "fl_ring",
- eth_dev->data->port_id, queue_id);
+ snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
+ eth_dev->data->port_id, queue_id,
+ fwevtq ? "fwq_ring" : "fl_ring");
snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
fl->desc = alloc_ring(fl->size, sizeof(__be64),
@@ -2144,9 +2142,8 @@ int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
/* Add status entries */
nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
- snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
- eth_dev->device->driver->name, "tx_ring",
- eth_dev->data->port_id, queue_id);
+ snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
+ eth_dev->data->port_id, queue_id, "tx_ring");
snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
txq->q.desc = alloc_ring(txq->q.size, sizeof(struct tx_desc),
@@ -2223,9 +2220,8 @@ int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
/* Add status entries */
nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
- snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
- eth_dev->device->driver->name, "ctrl_tx_ring",
- eth_dev->data->port_id, queue_id);
+ snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
+ eth_dev->data->port_id, queue_id, "ctrl_tx_ring");
snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
txq->q.desc = alloc_ring(txq->q.size, sizeof(struct tx_desc),