diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig index 288fa8ce53af0a..b513baf3cbb296 100644 --- a/drivers/net/ethernet/intel/Kconfig +++ b/drivers/net/ethernet/intel/Kconfig @@ -203,6 +203,7 @@ config IXGBE_IPSEC config IXGBEVF tristate "Intel(R) 10GbE PCI Express Virtual Function Ethernet support" depends on PCI_MSI + select LIBETH_XDP help This driver supports Intel(R) PCI Express virtual functions for the Intel(R) ixgbe driver. For more information on how to identify your diff --git a/drivers/net/ethernet/intel/ixgbevf/Makefile b/drivers/net/ethernet/intel/ixgbevf/Makefile index 01d3e892f3fa7b..cdae62f25fd926 100644 --- a/drivers/net/ethernet/intel/ixgbevf/Makefile +++ b/drivers/net/ethernet/intel/ixgbevf/Makefile @@ -6,5 +6,5 @@ obj-$(CONFIG_IXGBEVF) += ixgbevf.o -ixgbevf-y := vf.o mbx.o ethtool.o ixgbevf_main.o +ixgbevf-y := vf.o mbx.o ethtool.o ixgbevf_main.o ixgbevf_xsk.o ixgbevf-$(CONFIG_IXGBEVF_IPSEC) += ipsec.o diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h index e177d1d58696aa..afc927dd14381b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/defines.h +++ b/drivers/net/ethernet/intel/ixgbevf/defines.h @@ -71,7 +71,7 @@ typedef u32 ixgbe_link_speed; #define IXGBE_PSRTYPE_L2HDR 0x00001000 /* SRRCTL bit definitions */ -#define IXGBE_SRRCTL_BSIZEPKT_SHIFT 10 /* so many KBs */ +#define IXGBE_SRRCTL_BSIZEPKT_STEP 1024 #define IXGBE_SRRCTL_RDMTS_SHIFT 22 #define IXGBE_SRRCTL_RDMTS_MASK 0x01C00000 #define IXGBE_SRRCTL_DROP_EN 0x10000000 diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c index 537a60d5276f0f..387eae14ed44d7 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c +++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c @@ -72,13 +72,6 @@ static const char ixgbe_gstrings_test[][ETH_GSTRING_LEN] = { #define IXGBEVF_TEST_LEN (sizeof(ixgbe_gstrings_test) / ETH_GSTRING_LEN) -static const char ixgbevf_priv_flags_strings[][ETH_GSTRING_LEN] = { -#define IXGBEVF_PRIV_FLAGS_LEGACY_RX BIT(0) - "legacy-rx", -}; - -#define IXGBEVF_PRIV_FLAGS_STR_LEN ARRAY_SIZE(ixgbevf_priv_flags_strings) - static int ixgbevf_get_link_ksettings(struct net_device *netdev, struct ethtool_link_ksettings *cmd) { @@ -216,8 +209,6 @@ static void ixgbevf_get_drvinfo(struct net_device *netdev, strscpy(drvinfo->driver, ixgbevf_driver_name, sizeof(drvinfo->driver)); strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); - - drvinfo->n_priv_flags = IXGBEVF_PRIV_FLAGS_STR_LEN; } static void ixgbevf_get_ringparam(struct net_device *netdev, @@ -409,8 +400,6 @@ static int ixgbevf_get_sset_count(struct net_device *netdev, int stringset) return IXGBEVF_TEST_LEN; case ETH_SS_STATS: return IXGBEVF_STATS_LEN; - case ETH_SS_PRIV_FLAGS: - return IXGBEVF_PRIV_FLAGS_STR_LEN; default: return -EINVAL; } @@ -538,10 +527,6 @@ static void ixgbevf_get_strings(struct net_device *netdev, u32 stringset, p += ETH_GSTRING_LEN; } break; - case ETH_SS_PRIV_FLAGS: - memcpy(data, ixgbevf_priv_flags_strings, - IXGBEVF_PRIV_FLAGS_STR_LEN * ETH_GSTRING_LEN); - break; } } @@ -931,33 +916,71 @@ static int ixgbevf_get_rxfh(struct net_device *netdev, return err; } -static u32 ixgbevf_get_priv_flags(struct net_device *netdev) +static void ixgbevf_get_reported_q_num(u32 rx, u32 tx, u32 *dst_combined, + u32 *dst_rx, u32 *dst_tx) +{ + rx = min_t(u32, rx, num_online_cpus()); + *dst_combined = min_t(u32, rx, tx); + if (rx > tx) { + *dst_rx = rx - tx; + *dst_tx = 0; + } else { + *dst_tx = tx - rx; + *dst_rx = 0; + } +} + +static void ixgbevf_get_channels(struct net_device *netdev, + struct ethtool_channels *ch) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); - u32 priv_flags = 0; - if (adapter->flags & IXGBEVF_FLAGS_LEGACY_RX) - priv_flags |= IXGBEVF_PRIV_FLAGS_LEGACY_RX; + ixgbevf_get_reported_q_num(adapter->num_rx_queues, + adapter->num_tx_queues, + &ch->combined_count, &ch->rx_count, + &ch->tx_count); - return priv_flags; + ixgbevf_get_reported_q_num(adapter->q_caps.max_rxqs, + adapter->q_caps.max_txqs, + &ch->max_combined, &ch->max_rx, + &ch->max_tx); + + ch->max_other = NON_Q_VECTORS; + ch->other_count = NON_Q_VECTORS; } -static int ixgbevf_set_priv_flags(struct net_device *netdev, u32 priv_flags) +static int ixgbevf_set_channels(struct net_device *netdev, + struct ethtool_channels *ch) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); - unsigned int flags = adapter->flags; + u32 num_req = ch->combined_count; + bool was_up; + int err = 0; - flags &= ~IXGBEVF_FLAGS_LEGACY_RX; - if (priv_flags & IXGBEVF_PRIV_FLAGS_LEGACY_RX) - flags |= IXGBEVF_FLAGS_LEGACY_RX; + /* Do not change queue number if DCB is enabled */ + if (adapter->q_caps.num_tcs > 1) + return -EOPNOTSUPP; - if (flags != adapter->flags) { - adapter->flags = flags; + if (ch->rx_count || ch->tx_count || ch->other_count != NON_Q_VECTORS) + return -EINVAL; - /* reset interface to repopulate queues */ - if (netif_running(netdev)) - ixgbevf_reinit_locked(adapter); - } + if (num_req == adapter->num_rx_queues && + num_req == adapter->num_tx_queues) + return 0; + + adapter->num_req_qpairs = num_req; + + was_up = netif_running(netdev); + if (was_up) + ixgbevf_close(netdev); + + ixgbevf_clear_interrupt_scheme(adapter); + err = ixgbevf_init_interrupt_scheme(adapter); + if (err) + return err; + + if (was_up) + ixgbevf_open(netdev); return 0; } @@ -984,8 +1007,8 @@ static const struct ethtool_ops ixgbevf_ethtool_ops = { .get_rxfh_key_size = ixgbevf_get_rxfh_key_size, .get_rxfh = ixgbevf_get_rxfh, .get_link_ksettings = ixgbevf_get_link_ksettings, - .get_priv_flags = ixgbevf_get_priv_flags, - .set_priv_flags = ixgbevf_set_priv_flags, + .get_channels = ixgbevf_get_channels, + .set_channels = ixgbevf_set_channels, }; void ixgbevf_set_ethtool_ops(struct net_device *netdev) diff --git a/drivers/net/ethernet/intel/ixgbevf/ipsec.c b/drivers/net/ethernet/intel/ixgbevf/ipsec.c index fce35924ff8b51..706e123fa92284 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ipsec.c +++ b/drivers/net/ethernet/intel/ixgbevf/ipsec.c @@ -450,11 +450,11 @@ static const struct xfrmdev_ops ixgbevf_xfrmdev_ops = { * @first: current data packet * @itd: ipsec Tx data for later use in building context descriptor **/ -int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, +int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, struct libeth_sqe *first, struct ixgbevf_ipsec_tx_data *itd) { struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); + struct ixgbevf_skb_sqe_priv *sqe_priv = (void *)&first->priv; struct ixgbevf_ipsec *ipsec = adapter->ipsec; struct xfrm_state *xs; struct sec_path *sp; @@ -491,12 +491,12 @@ int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, itd->pfsa = tsa->pfsa - IXGBE_IPSEC_BASE_TX_INDEX; - first->tx_flags |= IXGBE_TX_FLAGS_IPSEC | IXGBE_TX_FLAGS_CSUM; + sqe_priv->tx_flags |= IXGBE_TX_FLAGS_IPSEC | IXGBE_TX_FLAGS_CSUM; if (xs->id.proto == IPPROTO_ESP) { itd->flags |= IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP | IXGBE_ADVTXD_TUCMD_L4T_TCP; - if (first->protocol == htons(ETH_P_IP)) + if (sqe_priv->protocol == htons(ETH_P_IP)) itd->flags |= IXGBE_ADVTXD_TUCMD_IPV4; /* The actual trailer length is authlen (16 bytes) plus diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index 516a6fdd23d076..ebdc3304e768eb 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "vf.h" @@ -23,35 +24,12 @@ #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) #define DESC_NEEDED (MAX_SKB_FRAGS + 4) -/* wrapper around a pointer to a socket buffer, - * so a DMA handle can be stored along with the buffer - */ -struct ixgbevf_tx_buffer { - union ixgbe_adv_tx_desc *next_to_watch; - unsigned long time_stamp; - union { - struct sk_buff *skb; - /* XDP uses address ptr on irq_clean */ - void *data; - }; - unsigned int bytecount; - unsigned short gso_segs; - __be16 protocol; - DEFINE_DMA_UNMAP_ADDR(dma); - DEFINE_DMA_UNMAP_LEN(len); +struct ixgbevf_skb_sqe_priv { u32 tx_flags; + __be16 protocol; }; - -struct ixgbevf_rx_buffer { - dma_addr_t dma; - struct page *page; -#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) - __u32 page_offset; -#else - __u16 page_offset; -#endif - __u16 pagecnt_bias; -}; +static_assert(sizeof(struct ixgbevf_skb_sqe_priv) <= + sizeof(typeof_member(struct libeth_sqe, priv))); struct ixgbevf_stats { u64 packets; @@ -72,12 +50,11 @@ struct ixgbevf_rx_queue_stats { }; enum ixgbevf_ring_state_t { - __IXGBEVF_RX_3K_BUFFER, - __IXGBEVF_RX_BUILD_SKB_ENABLED, __IXGBEVF_TX_DETECT_HANG, __IXGBEVF_HANG_CHECK_ARMED, __IXGBEVF_TX_XDP_RING, __IXGBEVF_TX_XDP_RING_PRIMED, + __IXGBEVF_RXTX_XSK_RING, }; #define ring_is_xdp(ring) \ @@ -87,24 +64,39 @@ enum ixgbevf_ring_state_t { #define clear_ring_xdp(ring) \ clear_bit(__IXGBEVF_TX_XDP_RING, &(ring)->state) +#define ring_is_xsk(ring) \ + test_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) +#define set_ring_xsk(ring) \ + set_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) +#define clear_ring_xsk(ring) \ + clear_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) + struct ixgbevf_ring { struct ixgbevf_ring *next; struct ixgbevf_q_vector *q_vector; /* backpointer to q_vector */ struct net_device *netdev; - struct bpf_prog *xdp_prog; - struct device *dev; + struct bpf_prog __rcu *xdp_prog; + union { + struct page_pool *pp; /* Rx and XDP rings */ + struct device *dev; /* Tx ring */ + }; void *desc; /* descriptor ring memory */ - dma_addr_t dma; /* phys. address of descriptor ring */ - unsigned int size; /* length in bytes */ + u32 truesize; /* Rx buffer full size */ + u32 hdr_truesize; /* Rx header buffer full size */ u16 count; /* amount of descriptors */ - u16 next_to_use; u16 next_to_clean; - u16 next_to_alloc; + u32 next_to_use; + u32 pending; /* Sent-not-completed descriptors */ union { - struct ixgbevf_tx_buffer *tx_buffer_info; - struct ixgbevf_rx_buffer *rx_buffer_info; + struct libeth_fqe *rx_fqes; + struct libeth_xdp_buff **xsk_fqes; + struct libeth_sqe *tx_sqes; + struct libeth_sqe *xdp_sqes; }; + struct libeth_xdpsq_lock xdpq_lock; + u32 cached_ntu; + u32 thresh; unsigned long state; struct ixgbevf_stats stats; struct u64_stats_sync syncp; @@ -112,16 +104,23 @@ struct ixgbevf_ring { struct ixgbevf_tx_queue_stats tx_stats; struct ixgbevf_rx_queue_stats rx_stats; }; + struct libeth_fqe *hdr_fqes; + struct page_pool *hdr_pp; struct xdp_rxq_info xdp_rxq; u64 hw_csum_rx_error; u8 __iomem *tail; - struct sk_buff *skb; /* holds the special value that gets the hardware register offset * associated with this ring, which is different for DCB and RSS modes */ u16 reg_idx; int queue_index; /* needed for multiqueue queue management */ + u32 rx_buf_len; + struct libeth_xdp_buff_stash xdp_stash; + struct libeth_xdp_buff *xsk_xdp_head; + unsigned int dma_size; /* length in bytes */ + dma_addr_t dma; /* phys. address of descriptor ring */ + struct xsk_buff_pool *xsk_pool; /* AF_XDP ZC rings */ } ____cacheline_internodealigned_in_smp; /* How many Rx Buffers do we bundle into one write to the hardware ? */ @@ -144,21 +143,17 @@ struct ixgbevf_ring { #define IXGBEVF_MIN_RXD 64 /* Supported Rx Buffer Sizes */ -#define IXGBEVF_RXBUFFER_256 256 /* Used for packet split */ -#define IXGBEVF_RXBUFFER_2048 2048 +#define IXGBEVF_RXBUFFER_256 256 #define IXGBEVF_RXBUFFER_3072 3072 #define IXGBEVF_RX_HDR_SIZE IXGBEVF_RXBUFFER_256 #define MAXIMUM_ETHERNET_VLAN_SIZE (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN) -#define IXGBEVF_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) -#if (PAGE_SIZE < 8192) -#define IXGBEVF_MAX_FRAME_BUILD_SKB \ - (SKB_WITH_OVERHEAD(IXGBEVF_RXBUFFER_2048) - IXGBEVF_SKB_PAD) -#else -#define IXGBEVF_MAX_FRAME_BUILD_SKB IXGBEVF_RXBUFFER_2048 -#endif +#define IXGBEVF_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) +#define IXGBEVF_RX_SRRCTL_BUF_SIZE(mtu) (ALIGN((mtu) + LIBETH_RX_LL_LEN, \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) #define IXGBE_TX_FLAGS_CSUM BIT(0) #define IXGBE_TX_FLAGS_VLAN BIT(1) @@ -169,43 +164,6 @@ struct ixgbevf_ring { #define IXGBE_TX_FLAGS_VLAN_PRIO_MASK 0x0000e000 #define IXGBE_TX_FLAGS_VLAN_SHIFT 16 -#define ring_uses_large_buffer(ring) \ - test_bit(__IXGBEVF_RX_3K_BUFFER, &(ring)->state) -#define set_ring_uses_large_buffer(ring) \ - set_bit(__IXGBEVF_RX_3K_BUFFER, &(ring)->state) -#define clear_ring_uses_large_buffer(ring) \ - clear_bit(__IXGBEVF_RX_3K_BUFFER, &(ring)->state) - -#define ring_uses_build_skb(ring) \ - test_bit(__IXGBEVF_RX_BUILD_SKB_ENABLED, &(ring)->state) -#define set_ring_build_skb_enabled(ring) \ - set_bit(__IXGBEVF_RX_BUILD_SKB_ENABLED, &(ring)->state) -#define clear_ring_build_skb_enabled(ring) \ - clear_bit(__IXGBEVF_RX_BUILD_SKB_ENABLED, &(ring)->state) - -static inline unsigned int ixgbevf_rx_bufsz(struct ixgbevf_ring *ring) -{ -#if (PAGE_SIZE < 8192) - if (ring_uses_large_buffer(ring)) - return IXGBEVF_RXBUFFER_3072; - - if (ring_uses_build_skb(ring)) - return IXGBEVF_MAX_FRAME_BUILD_SKB; -#endif - return IXGBEVF_RXBUFFER_2048; -} - -static inline unsigned int ixgbevf_rx_pg_order(struct ixgbevf_ring *ring) -{ -#if (PAGE_SIZE < 8192) - if (ring_uses_large_buffer(ring)) - return 1; -#endif - return 0; -} - -#define ixgbevf_rx_pg_size(_ring) (PAGE_SIZE << ixgbevf_rx_pg_order(_ring)) - #define check_for_tx_hang(ring) \ test_bit(__IXGBEVF_TX_DETECT_HANG, &(ring)->state) #define set_check_for_tx_hang(ring) \ @@ -275,6 +233,14 @@ static inline u16 ixgbevf_desc_unused(struct ixgbevf_ring *ring) return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1; } +static inline u16 ixgbevf_desc_used(struct ixgbevf_ring *ring) +{ + u16 ntc = ring->next_to_clean; + u16 ntu = ring->next_to_use; + + return ((ntu >= ntc) ? 0 : ring->count) + ntu - ntc; +} + static inline void ixgbevf_write_tail(struct ixgbevf_ring *ring, u32 value) { writel(value, ring->tail); @@ -300,6 +266,15 @@ static inline void ixgbevf_write_tail(struct ixgbevf_ring *ring, u32 value) #define IXGBEVF_RX_DMA_ATTR \ (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) +struct ixgbevf_q_caps { + u32 min_rxqs; + u32 max_rxqs; + u32 min_txqs; + u32 max_txqs; + u32 num_tcs; + u32 def_tc; +}; + /* board specific private data structure */ struct ixgbevf_adapter { /* this field must be first, see ixgbevf_process_skb_fields */ @@ -376,8 +351,8 @@ struct ixgbevf_adapter { u8 rss_indir_tbl[IXGBEVF_X550_VFRETA_SIZE]; u32 flags; bool link_state; - -#define IXGBEVF_FLAGS_LEGACY_RX BIT(1) + struct ixgbevf_q_caps q_caps; + u32 num_req_qpairs; #ifdef CONFIG_XFRM struct ixgbevf_ipsec *ipsec; @@ -396,6 +371,8 @@ enum ixbgevf_state_t { __IXGBEVF_QUEUE_RESET_REQUESTED, }; +#define IXGBEVF_FLAG_HSPLIT BIT(0) + enum ixgbevf_boards { board_82599_vf, board_82599_vf_hv, @@ -439,15 +416,31 @@ int ixgbevf_open(struct net_device *netdev); int ixgbevf_close(struct net_device *netdev); void ixgbevf_up(struct ixgbevf_adapter *adapter); void ixgbevf_down(struct ixgbevf_adapter *adapter); +void ixgbevf_flush_tx_queue(struct ixgbevf_ring *ring); +void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); +void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter); void ixgbevf_reset(struct ixgbevf_adapter *adapter); void ixgbevf_set_ethtool_ops(struct net_device *netdev); int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring); +void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter); +void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); int ixgbevf_setup_tx_resources(struct ixgbevf_ring *); +void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); void ixgbevf_free_rx_resources(struct ixgbevf_ring *); +void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring); +void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring); void ixgbevf_free_tx_resources(struct ixgbevf_ring *); +void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring); +void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring); void ixgbevf_update_stats(struct ixgbevf_adapter *adapter); +int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter); +void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter); int ethtool_ioctl(struct ifreq *ifr); extern void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector); @@ -459,8 +452,7 @@ void ixgbevf_ipsec_restore(struct ixgbevf_adapter *adapter); void ixgbevf_ipsec_rx(struct ixgbevf_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb); -int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, +int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, struct libeth_sqe *first, struct ixgbevf_ipsec_tx_data *itd); #else static inline void ixgbevf_init_ipsec_offload(struct ixgbevf_adapter *adapter) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index d5ce20f47def1f..1a386b345978bd 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -30,9 +30,12 @@ #include #include #include +#include #include -#include "ixgbevf.h" +#include "ixgbevf_txrx_lib.h" +#include "ixgbevf_xdp_lib.h" +#include "ixgbevf_xsk.h" const char ixgbevf_driver_name[] = "ixgbevf"; static const char ixgbevf_driver_string[] = @@ -82,6 +85,8 @@ static const struct pci_device_id ixgbevf_pci_tbl[] = { MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); +MODULE_IMPORT_NS("LIBETH"); +MODULE_IMPORT_NS("LIBETH_XDP"); MODULE_LICENSE("GPL v2"); #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) @@ -112,9 +117,6 @@ static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); -static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer); -static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *old_buff); static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) { @@ -252,13 +254,58 @@ static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter) * @netdev: network interface device structure * @txqueue: transmit queue hanging (unused) **/ -static void ixgbevf_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue) +static void ixgbevf_tx_timeout(struct net_device *netdev, unsigned int txqueue) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); + struct ixgbevf_ring *tx_ring = adapter->tx_ring[txqueue]; + u32 ntc = tx_ring->next_to_clean; + netdev_err(netdev, "Timeout info:\n"); + netdev_err(netdev, "ntu=%u, ntc=%u, rs_idx=%u, cached_ntu=%u\n", tx_ring->next_to_use, ntc, tx_ring->tx_sqes[ntc].rs_idx, tx_ring->cached_ntu); ixgbevf_tx_timeout_reset(adapter); } +static u16 ixgbevf_clean_tx_num(struct ixgbevf_ring *tx_ring, bool in_napi, + u16 to_clean, + struct libeth_sq_napi_stats *stats, + u16 *total_ipsec) +{ + bool xsk_ring = ring_is_xsk(tx_ring); + u16 ntc = tx_ring->next_to_clean; + struct xdp_frame_bulk xdp_bulk; + struct libeth_cq_pp cq = { + .ss = stats, + .dev = tx_ring->dev, + .napi = in_napi, + .bq = &xdp_bulk, + }; + u32 xsk_frames = 0; + + xdp_frame_bulk_init(&xdp_bulk); + for (int i = 0; i < to_clean; i++) { + struct ixgbevf_skb_sqe_priv *priv; + struct libeth_sqe *sqe; + + sqe = &tx_ring->tx_sqes[ntc]; + priv = (void *)&sqe->priv; + if (priv->tx_flags & IXGBE_TX_FLAGS_IPSEC && + likely(total_ipsec)) + (*total_ipsec)++; + + xsk_frames += xsk_ring && !sqe->type ? 1 : 0; + libeth_tx_complete_any(sqe, &cq); + + if (unlikely(++ntc == tx_ring->count)) + ntc = 0; + } + + xdp_flush_frame_bulk(&xdp_bulk); + if (xsk_frames) + xsk_tx_completed(tx_ring->xsk_pool, xsk_frames); + + return ntc; +} + /** * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes * @q_vector: board private structure @@ -269,110 +316,36 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *tx_ring, int napi_budget) { struct ixgbevf_adapter *adapter = q_vector->adapter; - struct ixgbevf_tx_buffer *tx_buffer; - union ixgbe_adv_tx_desc *tx_desc; - unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0; - unsigned int budget = tx_ring->count / 2; - unsigned int i = tx_ring->next_to_clean; + u16 budget = tx_ring->count / 2, to_clean, ntc; + struct libeth_sq_napi_stats stats = { }; + u16 total_ipsec; if (test_bit(__IXGBEVF_DOWN, &adapter->state)) return true; - tx_buffer = &tx_ring->tx_buffer_info[i]; - tx_desc = IXGBEVF_TX_DESC(tx_ring, i); - i -= tx_ring->count; - - do { - union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; - - /* if next_to_watch is not set then there is no work pending */ - if (!eop_desc) - break; - - /* prevent any other reads prior to eop_desc */ - smp_rmb(); - - /* if DD is not set pending work has not been completed */ - if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) - break; - - /* clear next_to_watch to prevent false hangs */ - tx_buffer->next_to_watch = NULL; - - /* update the statistics for this packet */ - total_bytes += tx_buffer->bytecount; - total_packets += tx_buffer->gso_segs; - if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC) - total_ipsec++; - - /* free the skb */ - if (ring_is_xdp(tx_ring)) - page_frag_free(tx_buffer->data); - else - napi_consume_skb(tx_buffer->skb, napi_budget); - - /* unmap skb header data */ - dma_unmap_single(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); - - /* clear tx_buffer data */ - dma_unmap_len_set(tx_buffer, len, 0); - - /* unmap remaining buffers */ - while (tx_desc != eop_desc) { - tx_buffer++; - tx_desc++; - i++; - if (unlikely(!i)) { - i -= tx_ring->count; - tx_buffer = tx_ring->tx_buffer_info; - tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); - } - - /* unmap any remaining paged data */ - if (dma_unmap_len(tx_buffer, len)) { - dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buffer, len, 0); - } - } - - /* move us one more past the eop_desc for start of next pkt */ - tx_buffer++; - tx_desc++; - i++; - if (unlikely(!i)) { - i -= tx_ring->count; - tx_buffer = tx_ring->tx_buffer_info; - tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); - } + to_clean = ixgbevf_desc_used(tx_ring); + to_clean = ixgbevf_tx_get_num_sent(tx_ring, min_t(u16, budget, to_clean)); + budget = budget > to_clean ? budget - to_clean : 0; + ntc = ixgbevf_clean_tx_num(tx_ring, true, to_clean, &stats, &total_ipsec); - /* issue prefetch for next Tx descriptor */ - prefetch(tx_desc); - - /* update budget accounting */ - budget--; - } while (likely(budget)); + smp_wmb(); - i += tx_ring->count; - tx_ring->next_to_clean = i; + tx_ring->next_to_clean = ntc; u64_stats_update_begin(&tx_ring->syncp); - tx_ring->stats.bytes += total_bytes; - tx_ring->stats.packets += total_packets; + tx_ring->stats.bytes += stats.bytes; + tx_ring->stats.packets += stats.packets; u64_stats_update_end(&tx_ring->syncp); - q_vector->tx.total_bytes += total_bytes; - q_vector->tx.total_packets += total_packets; + q_vector->tx.total_bytes += stats.bytes; + q_vector->tx.total_packets += stats.packets; adapter->tx_ipsec += total_ipsec; if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) { struct ixgbe_hw *hw = &adapter->hw; union ixgbe_adv_tx_desc *eop_desc; + u32 rs_idx; - eop_desc = tx_ring->tx_buffer_info[i].next_to_watch; + rs_idx = tx_ring->tx_sqes[ntc].rs_idx; + eop_desc = rs_idx ? IXGBEVF_TX_DESC(tx_ring, rs_idx - 1) : NULL; pr_err("Detected Tx Unit Hang%s\n" " Tx Queue <%d>\n" @@ -381,20 +354,16 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, " next_to_clean <%x>\n" "tx_buffer_info[next_to_clean]\n" " next_to_watch <%p>\n" - " eop_desc->wb.status <%x>\n" - " time_stamp <%lx>\n" - " jiffies <%lx>\n", + " eop_desc->wb.status <%x>\n", ring_is_xdp(tx_ring) ? " XDP" : "", tx_ring->queue_index, IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)), IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)), - tx_ring->next_to_use, i, - eop_desc, (eop_desc ? eop_desc->wb.status : 0), - tx_ring->tx_buffer_info[i].time_stamp, jiffies); + tx_ring->next_to_use, ntc, + eop_desc, (eop_desc ? eop_desc->wb.status : 0)); - if (!ring_is_xdp(tx_ring)) - netif_stop_subqueue(tx_ring->netdev, - tx_ring->queue_index); + netif_stop_subqueue(tx_ring->netdev, + tx_ring->queue_index); /* schedule immediate reset if we believe we hung */ ixgbevf_tx_timeout_reset(adapter); @@ -402,11 +371,8 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, return true; } - if (ring_is_xdp(tx_ring)) - return !!budget; - #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) - if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && + if (unlikely(stats.packets && netif_carrier_ok(tx_ring->netdev) && (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { /* Make sure that anybody stopping the queue after this * sees the new next_to_clean. @@ -425,227 +391,6 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, return !!budget; } -/** - * ixgbevf_rx_skb - Helper function to determine proper Rx method - * @q_vector: structure containing interrupt and ring information - * @skb: packet to send up - **/ -static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, - struct sk_buff *skb) -{ - napi_gro_receive(&q_vector->napi, skb); -} - -#define IXGBE_RSS_L4_TYPES_MASK \ - ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ - (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ - (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ - (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) - -static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - u16 rss_type; - - if (!(ring->netdev->features & NETIF_F_RXHASH)) - return; - - rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & - IXGBE_RXDADV_RSSTYPE_MASK; - - if (!rss_type) - return; - - skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), - (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? - PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); -} - -/** - * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum - * @ring: structure containig ring specific data - * @rx_desc: current Rx descriptor being processed - * @skb: skb currently being received and modified - **/ -static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - skb_checksum_none_assert(skb); - - /* Rx csum disabled */ - if (!(ring->netdev->features & NETIF_F_RXCSUM)) - return; - - /* if IP and error */ - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && - ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { - ring->rx_stats.csum_err++; - return; - } - - if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) - return; - - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { - ring->rx_stats.csum_err++; - return; - } - - /* It must be a TCP or UDP packet with a valid checksum */ - skb->ip_summed = CHECKSUM_UNNECESSARY; -} - -/** - * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor - * @rx_ring: rx descriptor ring packet is being transacted on - * @rx_desc: pointer to the EOP Rx descriptor - * @skb: pointer to current skb being populated - * - * This function checks the ring, descriptor, and packet information in - * order to populate the checksum, VLAN, protocol, and other fields within - * the skb. - **/ -static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - ixgbevf_rx_hash(rx_ring, rx_desc, skb); - ixgbevf_rx_checksum(rx_ring, rx_desc, skb); - - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { - u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); - unsigned long *active_vlans = netdev_priv(rx_ring->netdev); - - if (test_bit(vid & VLAN_VID_MASK, active_vlans)) - __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); - } - - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP)) - ixgbevf_ipsec_rx(rx_ring, rx_desc, skb); - - skb->protocol = eth_type_trans(skb, rx_ring->netdev); -} - -static -struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring, - const unsigned int size) -{ - struct ixgbevf_rx_buffer *rx_buffer; - - rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - prefetchw(rx_buffer->page); - - /* we are reusing so sync this buffer for CPU use */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - size, - DMA_FROM_DEVICE); - - rx_buffer->pagecnt_bias--; - - return rx_buffer; -} - -static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - struct sk_buff *skb) -{ - if (ixgbevf_can_reuse_rx_page(rx_buffer)) { - /* hand second half of page back to the ring */ - ixgbevf_reuse_rx_page(rx_ring, rx_buffer); - } else { - if (IS_ERR(skb)) - /* We are not reusing the buffer so unmap it and free - * any references we are holding to it - */ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - ixgbevf_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBEVF_RX_DMA_ATTR); - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); - } - - /* clear contents of rx_buffer */ - rx_buffer->page = NULL; -} - -/** - * ixgbevf_is_non_eop - process handling of non-EOP buffers - * @rx_ring: Rx ring being processed - * @rx_desc: Rx descriptor for current buffer - * - * This function updates next to clean. If the buffer is an EOP buffer - * this function exits returning false, otherwise it will place the - * sk_buff in the next buffer to be chained and return true indicating - * that this is in fact a non-EOP buffer. - **/ -static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc) -{ - u32 ntc = rx_ring->next_to_clean + 1; - - /* fetch, update, and store next to clean */ - ntc = (ntc < rx_ring->count) ? ntc : 0; - rx_ring->next_to_clean = ntc; - - prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); - - if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) - return false; - - return true; -} - -static inline unsigned int ixgbevf_rx_offset(struct ixgbevf_ring *rx_ring) -{ - return ring_uses_build_skb(rx_ring) ? IXGBEVF_SKB_PAD : 0; -} - -static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *bi) -{ - struct page *page = bi->page; - dma_addr_t dma; - - /* since we are recycling buffers we should seldom need to alloc */ - if (likely(page)) - return true; - - /* alloc new page for storage */ - page = dev_alloc_pages(ixgbevf_rx_pg_order(rx_ring)); - if (unlikely(!page)) { - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, - ixgbevf_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); - - /* if mapping failed free memory back to system since - * there isn't much point in holding memory we can't use - */ - if (dma_mapping_error(rx_ring->dev, dma)) { - __free_pages(page, ixgbevf_rx_pg_order(rx_ring)); - - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - bi->dma = dma; - bi->page = page; - bi->page_offset = ixgbevf_rx_offset(rx_ring); - bi->pagecnt_bias = 1; - rx_ring->rx_stats.alloc_rx_page++; - - return true; -} - /** * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on @@ -655,39 +400,48 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, u16 cleaned_count) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbevf_rx_buffer *bi; - unsigned int i = rx_ring->next_to_use; + const struct libeth_fq_fp fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + .truesize = rx_ring->truesize, + .count = rx_ring->count, + }; + const struct libeth_fq_fp hdr_fq = { + .pp = rx_ring->hdr_pp, + .fqes = rx_ring->hdr_fqes, + .truesize = rx_ring->hdr_truesize, + .count = rx_ring->count, + }; + u16 ntu = rx_ring->next_to_use; - /* nothing to do or no valid netdev defined */ - if (!cleaned_count || !rx_ring->netdev) + /* nothing to do or page pool is not present */ + if (unlikely(!cleaned_count || !fq.pp)) return; - rx_desc = IXGBEVF_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_buffer_info[i]; - i -= rx_ring->count; + rx_desc = IXGBEVF_RX_DESC(rx_ring, ntu); do { - if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) + dma_addr_t addr; + + addr = libeth_rx_alloc(&fq, ntu); + if (addr == DMA_MAPPING_ERROR) break; - /* sync the buffer for use by the device */ - dma_sync_single_range_for_device(rx_ring->dev, bi->dma, - bi->page_offset, - ixgbevf_rx_bufsz(rx_ring), - DMA_FROM_DEVICE); + rx_desc->read.pkt_addr = cpu_to_le64(addr); - /* Refresh the desc even if pkt_addr didn't change - * because each write-back erases this info. - */ - rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); + if (hdr_fq.pp) { + addr = libeth_rx_alloc(&hdr_fq, ntu); + if (addr == DMA_MAPPING_ERROR) { + libeth_rx_recycle_slow(fq.fqes[ntu].netmem); + break; + } + } rx_desc++; - bi++; - i++; - if (unlikely(!i)) { + ntu++; + if (unlikely(ntu == fq.count)) { rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_buffer_info; - i -= rx_ring->count; + ntu = 0; } /* clear the length for the next_to_use descriptor */ @@ -696,14 +450,9 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, cleaned_count--; } while (cleaned_count); - i += rx_ring->count; - - if (rx_ring->next_to_use != i) { + if (likely(rx_ring->next_to_use != ntu)) { /* record the next descriptor to use */ - rx_ring->next_to_use = i; - - /* update next to alloc since we have filled the ring */ - rx_ring->next_to_alloc = i; + rx_ring->next_to_use = ntu; /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only @@ -711,206 +460,8 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, * such as IA-64). */ wmb(); - ixgbevf_write_tail(rx_ring, i); - } -} - -/** - * ixgbevf_cleanup_headers - Correct corrupted or empty headers - * @rx_ring: rx descriptor ring packet is being transacted on - * @rx_desc: pointer to the EOP Rx descriptor - * @skb: pointer to current skb being fixed - * - * Check for corrupted packet headers caused by senders on the local L2 - * embedded NIC switch not setting up their Tx Descriptors right. These - * should be very rare. - * - * Also address the case where we are pulling data in on pages only - * and as such no data is present in the skb header. - * - * In addition if skb is not at least 60 bytes we need to pad it so that - * it is large enough to qualify as a valid Ethernet frame. - * - * Returns true if an error was encountered and skb was freed. - **/ -static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - /* verify that the packet does not have any known errors */ - if (unlikely(ixgbevf_test_staterr(rx_desc, - IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { - struct net_device *netdev = rx_ring->netdev; - - if (!(netdev->features & NETIF_F_RXALL)) { - dev_kfree_skb_any(skb); - return true; - } - } - - /* if eth_skb_pad returns an error the skb was freed */ - if (eth_skb_pad(skb)) - return true; - - return false; -} - -/** - * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring - * @rx_ring: rx descriptor ring to store buffers on - * @old_buff: donor buffer to have page reused - * - * Synchronizes page for reuse by the adapter - **/ -static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *old_buff) -{ - struct ixgbevf_rx_buffer *new_buff; - u16 nta = rx_ring->next_to_alloc; - - new_buff = &rx_ring->rx_buffer_info[nta]; - - /* update, and store next to alloc */ - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - /* transfer page from old buffer to new buffer */ - new_buff->page = old_buff->page; - new_buff->dma = old_buff->dma; - new_buff->page_offset = old_buff->page_offset; - new_buff->pagecnt_bias = old_buff->pagecnt_bias; -} - -static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer) -{ - unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; - struct page *page = rx_buffer->page; - - /* avoid re-using remote and pfmemalloc pages */ - if (!dev_page_is_reusable(page)) - return false; - -#if (PAGE_SIZE < 8192) - /* if we are only owner of page we can reuse it */ - if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) - return false; -#else -#define IXGBEVF_LAST_OFFSET \ - (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBEVF_RXBUFFER_2048) - - if (rx_buffer->page_offset > IXGBEVF_LAST_OFFSET) - return false; - -#endif - - /* If we have drained the page fragment pool we need to update - * the pagecnt_bias and page count so that we fully restock the - * number of references the driver holds. - */ - if (unlikely(!pagecnt_bias)) { - page_ref_add(page, USHRT_MAX); - rx_buffer->pagecnt_bias = USHRT_MAX; - } - - return true; -} - -/** - * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff - * @rx_ring: rx descriptor ring to transact packets on - * @rx_buffer: buffer containing page to add - * @skb: sk_buff to place the data into - * @size: size of buffer to be added - * - * This function will add the data contained in rx_buffer->page to the skb. - **/ -static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - struct sk_buff *skb, - unsigned int size) -{ -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = ring_uses_build_skb(rx_ring) ? - SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) : - SKB_DATA_ALIGN(size); -#endif - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, - rx_buffer->page_offset, size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif -} - -static -struct sk_buff *ixgbevf_construct_skb(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int size = xdp->data_end - xdp->data; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); -#endif - unsigned int headlen; - struct sk_buff *skb; - - /* prefetch first cache line of first page */ - net_prefetch(xdp->data); - - /* Note, we get here by enabling legacy-rx via: - * - * ethtool --set-priv-flags legacy-rx on - * - * In this mode, we currently get 0 extra XDP headroom as - * opposed to having legacy-rx off, where we process XDP - * packets going to stack via ixgbevf_build_skb(). - * - * For ixgbevf_construct_skb() mode it means that the - * xdp->data_meta will always point to xdp->data, since - * the helper cannot expand the head. Should this ever - * changed in future for legacy-rx mode on, then lets also - * add xdp->data_meta handling here. - */ - - /* allocate a skb to store the frags */ - skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBEVF_RX_HDR_SIZE); - if (unlikely(!skb)) - return NULL; - - /* Determine available headroom for copy */ - headlen = size; - if (headlen > IXGBEVF_RX_HDR_SIZE) - headlen = eth_get_headlen(skb->dev, xdp->data, - IXGBEVF_RX_HDR_SIZE); - - /* align pull length to size of long to optimize memcpy performance */ - memcpy(__skb_put(skb, headlen), xdp->data, - ALIGN(headlen, sizeof(long))); - - /* update all of the pointers */ - size -= headlen; - if (size) { - skb_add_rx_frag(skb, 0, rx_buffer->page, - (xdp->data + headlen) - - page_address(rx_buffer->page), - size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - } else { - rx_buffer->pagecnt_bias++; + ixgbevf_write_tail(rx_ring, ntu); } - - return skb; } static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, @@ -921,219 +472,128 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); } -static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) +void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring) { - unsigned int metasize = xdp->data - xdp->data_meta; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + - SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); -#endif - struct sk_buff *skb; - - /* Prefetch first cache line of first page. If xdp->data_meta - * is unused, this points to xdp->data, otherwise, we likely - * have a consumer accessing first few bytes of meta data, - * and then actual data. - */ - net_prefetch(xdp->data_meta); - - /* build an skb around the page buffer */ - skb = napi_build_skb(xdp->data_hard_start, truesize); - if (unlikely(!skb)) - return NULL; - - /* update pointers within the skb to store the data */ - skb_reserve(skb, xdp->data - xdp->data_hard_start); - __skb_put(skb, xdp->data_end - xdp->data); - if (metasize) - skb_metadata_set(skb, metasize); - - /* update buffer offset */ -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - - return skb; + ixgbevf_clean_xdp_num(xdp_ring, false, xdp_ring->pending); + libeth_xdpsq_put(&xdp_ring->xdpq_lock, xdp_ring->netdev); } -#define IXGBEVF_XDP_PASS 0 -#define IXGBEVF_XDP_CONSUMED 1 -#define IXGBEVF_XDP_TX 2 - -static int ixgbevf_xmit_xdp_ring(struct ixgbevf_ring *ring, - struct xdp_buff *xdp) +static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, + u64 priv) { - struct ixgbevf_tx_buffer *tx_buffer; - union ixgbe_adv_tx_desc *tx_desc; - u32 len, cmd_type; - dma_addr_t dma; - u16 i; - - len = xdp->data_end - xdp->data; + union ixgbe_adv_tx_desc *tx_desc = + &((union ixgbe_adv_tx_desc *)sq->descs)[i]; - if (unlikely(!ixgbevf_desc_unused(ring))) - return IXGBEVF_XDP_CONSUMED; + u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS | + desc.len; - dma = dma_map_single(ring->dev, xdp->data, len, DMA_TO_DEVICE); - if (dma_mapping_error(ring->dev, dma)) - return IXGBEVF_XDP_CONSUMED; + if (desc.flags & LIBETH_XDP_TX_LAST) + cmd_type |= IXGBE_TXD_CMD_EOP; - /* record the location of the first descriptor for this packet */ - i = ring->next_to_use; - tx_buffer = &ring->tx_buffer_info[i]; - - dma_unmap_len_set(tx_buffer, len, len); - dma_unmap_addr_set(tx_buffer, dma, dma); - tx_buffer->data = xdp->data; - tx_buffer->bytecount = len; - tx_buffer->gso_segs = 1; - tx_buffer->protocol = 0; - - /* Populate minimal context descriptor that will provide for the - * fact that we are expected to process Ethernet frames. - */ - if (!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state)) { - struct ixgbe_adv_tx_context_desc *context_desc; + if (desc.flags & LIBETH_XDP_TX_FIRST) { + struct libeth_sqe *sqe = &sq->sqes[i]; + struct skb_shared_info *sinfo; + u16 full_len = desc.len; - set_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); - - context_desc = IXGBEVF_TX_CTXTDESC(ring, 0); - context_desc->vlan_macip_lens = - cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT); - context_desc->fceof_saidx = 0; - context_desc->type_tucmd_mlhl = - cpu_to_le32(IXGBE_TXD_CMD_DEXT | - IXGBE_ADVTXD_DTYP_CTXT); - context_desc->mss_l4len_idx = 0; + if (desc.flags & LIBETH_XDP_TX_MULTI) { + sinfo = sqe->type == LIBETH_SQE_XDP_TX ? + sqe->sinfo : + xdp_get_shared_info_from_frame(sqe->xdpf); + full_len += sinfo->xdp_frags_size; + } - i = 1; + tx_desc->read.olinfo_status = + cpu_to_le32((full_len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); } - /* put descriptor type bits */ - cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS; - cmd_type |= len | IXGBE_TXD_CMD; - - tx_desc = IXGBEVF_TX_DESC(ring, i); - tx_desc->read.buffer_addr = cpu_to_le64(dma); - + tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - tx_desc->read.olinfo_status = - cpu_to_le32((len << IXGBE_ADVTXD_PAYLEN_SHIFT) | - IXGBE_ADVTXD_CC); - - /* Avoid any potential race with cleanup */ - smp_wmb(); +} - /* set next_to_watch value indicating a packet is present */ - i++; - if (i == ring->count) - i = 0; +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xdp_flush_tx, ixgbevf_prep_xdp_sq, + ixgbevf_xdp_xmit_desc); +__LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xdp_flush_tx_shared, + ixgbevf_prep_tx_sq, ixgbevf_xdp_xmit_desc, + ixgbevf_xdp_tx_unprep, xdp); +LIBETH_XDP_DEFINE_FLUSH_XMIT(static ixgbevf_xdp_flush_xmit, ixgbevf_prep_xdp_sq, + ixgbevf_xdp_xmit_desc); +__LIBETH_XDP_DEFINE_FLUSH_XMIT(static ixgbevf_xdp_flush_xmit_shared, + ixgbevf_prep_tx_sq, ixgbevf_xdp_xmit_desc, + ixgbevf_xdp_tx_unprep); +LIBETH_XDP_DEFINE_END(); - tx_buffer->next_to_watch = tx_desc; - ring->next_to_use = i; +static bool ixgbevf_xdp_flush_common(struct libeth_xdp_tx_bulk *bq, u32 flags) +{ + const struct ixgbevf_ring *ring = bq->xdpsq; - return IXGBEVF_XDP_TX; + if (test_bit(__IXGBEVF_TX_XDP_RING, &ring->state)) + return ixgbevf_xdp_flush_tx(bq, flags); + else + return ixgbevf_xdp_flush_tx_shared(bq, flags); } -static int ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *rx_ring, - struct xdp_buff *xdp) -{ - int result = IXGBEVF_XDP_PASS; - struct ixgbevf_ring *xdp_ring; - struct bpf_prog *xdp_prog; - u32 act; - - xdp_prog = READ_ONCE(rx_ring->xdp_prog); +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_RUN_PROG(static ixgbevf_xdp_run_prog, ixgbevf_xdp_flush_common); +LIBETH_XDP_DEFINE_FINALIZE(static ixgbevf_xdp_finalize_xdp_napi, + ixgbevf_xdp_flush_common, ixgbevf_xdp_rs_and_bump); +LIBETH_XDP_DEFINE_END(); - if (!xdp_prog) - goto xdp_out; +static u32 ixgbevf_rx_hsplit_wa(const struct libeth_fqe *hdr, + struct libeth_fqe *buf, u32 data_len) +{ + u32 copy = data_len <= L1_CACHE_BYTES ? data_len : ETH_HLEN; + struct page *hdr_page, *buf_page; + const void *src; + void *dst; - act = bpf_prog_run_xdp(xdp_prog, xdp); - switch (act) { - case XDP_PASS: - break; - case XDP_TX: - xdp_ring = adapter->xdp_ring[rx_ring->queue_index]; - result = ixgbevf_xmit_xdp_ring(xdp_ring, xdp); - if (result == IXGBEVF_XDP_CONSUMED) - goto out_failure; - break; - default: - bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); - fallthrough; - case XDP_ABORTED: -out_failure: - trace_xdp_exception(rx_ring->netdev, xdp_prog, act); - fallthrough; /* handle aborts by dropping packet */ - case XDP_DROP: - result = IXGBEVF_XDP_CONSUMED; - break; - } -xdp_out: - return result; -} + if (unlikely(netmem_is_net_iov(buf->netmem)) || + !libeth_rx_sync_for_cpu(buf, copy)) + return 0; -static unsigned int ixgbevf_rx_frame_truesize(struct ixgbevf_ring *rx_ring, - unsigned int size) -{ - unsigned int truesize; + hdr_page = __netmem_to_page(hdr->netmem); + buf_page = __netmem_to_page(buf->netmem); -#if (PAGE_SIZE < 8192) - truesize = ixgbevf_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ -#else - truesize = ring_uses_build_skb(rx_ring) ? - SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : - SKB_DATA_ALIGN(size); -#endif - return truesize; -} + dst = page_address(hdr_page) + hdr->offset + + pp_page_to_nmdesc(hdr_page)->pp->p.offset; + src = page_address(buf_page) + buf->offset + + pp_page_to_nmdesc(buf_page)->pp->p.offset; -static void ixgbevf_rx_buffer_flip(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - unsigned int size) -{ - unsigned int truesize = ixgbevf_rx_frame_truesize(rx_ring, size); + memcpy(dst, src, LARGEST_ALIGN(copy)); + buf->offset += copy; -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif + return copy; } static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *rx_ring, int budget) { - unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0; + unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbevf_adapter *adapter = q_vector->adapter; u16 cleaned_count = ixgbevf_desc_unused(rx_ring); - struct sk_buff *skb = rx_ring->skb; - bool xdp_xmit = false; - struct xdp_buff xdp; - int xdp_res = 0; - - /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ -#if (PAGE_SIZE < 8192) - frame_sz = ixgbevf_rx_frame_truesize(rx_ring, 0); -#endif - xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); + LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); + LIBETH_XDP_ONSTACK_BUFF(xdp); + + libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); + if (adapter->num_xdp_queues) + libeth_xdp_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, + adapter->netdev, adapter->xdp_ring, + adapter->num_xdp_queues); + else + libeth_xdp_tx_init_bulk_shared(&xdp_tx_bulk, rx_ring->xdp_prog, + adapter->netdev, + adapter->tx_ring, + adapter->num_tx_queues); while (likely(total_rx_packets < budget)) { - struct ixgbevf_rx_buffer *rx_buffer; union ixgbe_adv_rx_desc *rx_desc; + struct libeth_fqe *rx_buffer; + struct sk_buff *skb; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -1144,7 +604,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); size = le16_to_cpu(rx_desc->wb.upper.length); - if (!size) + if (unlikely(!size)) break; /* This memory barrier is needed to keep us from reading @@ -1153,73 +613,62 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, */ rmb(); - rx_buffer = ixgbevf_get_rx_buffer(rx_ring, size); + rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; - /* retrieve a buffer from the ring */ - if (!skb) { - unsigned int offset = ixgbevf_rx_offset(rx_ring); - unsigned char *hard_start; + if (unlikely(rx_ring->hdr_pp)) { + struct libeth_fqe *hdr_buff; + unsigned int hdr_size = 0; - hard_start = page_address(rx_buffer->page) + - rx_buffer->page_offset - offset; - xdp_prepare_buff(&xdp, hard_start, offset, size, true); -#if (PAGE_SIZE > 4096) - /* At larger PAGE_SIZE, frame_sz depend on len size */ - xdp.frame_sz = ixgbevf_rx_frame_truesize(rx_ring, size); -#endif - xdp_res = ixgbevf_run_xdp(adapter, rx_ring, &xdp); - } + hdr_buff = &rx_ring->hdr_fqes[rx_ring->next_to_clean]; - if (xdp_res) { - if (xdp_res == IXGBEVF_XDP_TX) { - xdp_xmit = true; - ixgbevf_rx_buffer_flip(rx_ring, rx_buffer, - size); - } else { - rx_buffer->pagecnt_bias++; + if (!xdp->data) { + hdr_size = ixgbevf_rx_hsplit_wa(hdr_buff, + rx_buffer, + size); + size -= hdr_size ? : size; } - total_rx_packets++; - total_rx_bytes += size; - } else if (skb) { - ixgbevf_add_rx_frag(rx_ring, rx_buffer, skb, size); - } else if (ring_uses_build_skb(rx_ring)) { - skb = ixgbevf_build_skb(rx_ring, rx_buffer, - &xdp, rx_desc); - } else { - skb = ixgbevf_construct_skb(rx_ring, rx_buffer, - &xdp, rx_desc); - } - /* exit if we failed to retrieve a buffer */ - if (!xdp_res && !skb) { - rx_ring->rx_stats.alloc_rx_buff_failed++; - rx_buffer->pagecnt_bias++; - break; + libeth_xdp_process_buff(xdp, hdr_buff, hdr_size); } - ixgbevf_put_rx_buffer(rx_ring, rx_buffer, skb); - cleaned_count++; + libeth_xdp_process_buff(xdp, rx_buffer, size); + cleaned_count++; /* fetch next buffer in frame if non-eop */ if (ixgbevf_is_non_eop(rx_ring, rx_desc)) continue; + total_rx_packets++; + total_rx_bytes += xdp_get_buff_len(&xdp->base); + + if (xdp_tx_bulk.prog && + !ixgbevf_xdp_run_prog(xdp, &xdp_tx_bulk)) + continue; + + skb = xdp_build_skb_from_buff(&xdp->base); + + /* exit if we failed to retrieve a buffer */ + if (unlikely(!skb)) { + libeth_xdp_return_buff_slow(xdp); + rx_ring->rx_stats.alloc_rx_buff_failed++; + break; + } + + xdp->data = NULL; + /* verify the packet layout is correct */ - if (xdp_res || ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { + if (unlikely(ixgbevf_cleanup_headers(rx_ring, rx_desc, skb))) { skb = NULL; continue; } - /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; - /* Workaround hardware that can't do proper VEPA multicast * source pruning. */ - if ((skb->pkt_type == PACKET_BROADCAST || - skb->pkt_type == PACKET_MULTICAST) && - ether_addr_equal(rx_ring->netdev->dev_addr, - eth_hdr(skb)->h_source)) { + if (unlikely((skb->pkt_type == PACKET_BROADCAST || + skb->pkt_type == PACKET_MULTICAST) && + ether_addr_equal(rx_ring->netdev->dev_addr, + eth_hdr(skb)->h_source))) { dev_kfree_skb_irq(skb); continue; } @@ -1227,28 +676,13 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* populate checksum, VLAN, and protocol */ ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); - ixgbevf_rx_skb(q_vector, skb); - - /* reset skb pointer */ - skb = NULL; - - /* update budget accounting */ - total_rx_packets++; + napi_gro_receive(&q_vector->napi, skb); } /* place incomplete frames back on ring for completion */ - rx_ring->skb = skb; + libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); - if (xdp_xmit) { - struct ixgbevf_ring *xdp_ring = - adapter->xdp_ring[rx_ring->queue_index]; - - /* Force memory writes to complete before letting h/w - * know there are new descriptors to fetch. - */ - wmb(); - ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); - } + ixgbevf_xdp_finalize_xdp_napi(&xdp_tx_bulk); u64_stats_update_begin(&rx_ring->syncp); rx_ring->stats.packets += total_rx_packets; @@ -1260,6 +694,36 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, return total_rx_packets; } +static int ixgbevf_xdp_xmit(struct net_device *dev, int n, + struct xdp_frame **frames, u32 flags) +{ + struct ixgbevf_adapter *adapter = netdev_priv(dev); + + if (unlikely(test_bit(__IXGBEVF_DOWN, &adapter->state))) + return -ENETDOWN; + + if (unlikely(!READ_ONCE(adapter->xdp_prog))) + return -ENXIO; + + if (adapter->num_xdp_queues) + return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, + adapter->xdp_ring, + adapter->num_xdp_queues, + ixgbevf_xdp_flush_xmit, + ixgbevf_xdp_rs_and_bump); + else { + LIBETH_XDP_ONSTACK_BULK(xdp_bulk); + + libeth_xdp_xmit_init_bulk_shared(&xdp_bulk, dev, + adapter->tx_ring, + adapter->num_tx_queues); + + return __libeth_xdp_xmit_do_bulk(&xdp_bulk, frames, n, flags, + ixgbevf_xdp_flush_xmit_shared, + ixgbevf_xdp_rs_and_bump); + } +} + /** * ixgbevf_poll - NAPI polling calback * @napi: napi struct with our devices info in it @@ -1278,8 +742,13 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) bool clean_complete = true; ixgbevf_for_each_ring(ring, q_vector->tx) { - if (!ixgbevf_clean_tx_irq(q_vector, ring, budget)) - clean_complete = false; + if (!ring_is_xdp(ring)) + clean_complete &= + ixgbevf_clean_tx_irq(q_vector, ring, budget); + if (ring_is_xsk(ring)) + clean_complete &= + ixgbevf_clean_xsk_tx_irq(q_vector, ring, + budget); } if (budget <= 0) @@ -1294,7 +763,10 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) per_ring_budget = budget; ixgbevf_for_each_ring(ring, q_vector->rx) { - int cleaned = ixgbevf_clean_rx_irq(q_vector, ring, + int cleaned = ring_is_xsk(ring) ? + ixgbevf_clean_xsk_rx_irq(q_vector, ring, + per_ring_budget) : + ixgbevf_clean_rx_irq(q_vector, ring, per_ring_budget); work_done += cleaned; if (cleaned >= per_ring_budget) @@ -1666,7 +1138,7 @@ static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) * ixgbevf_irq_enable - Enable default interrupt generation settings * @adapter: board private structure **/ -static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) +void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; @@ -1675,6 +1147,26 @@ static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); } +/** + * ixgbevf_xsk_pool_from_q - get ZC XSK buffer pool bound to a queue ID + * @ring: Rx or Tx ring + * + * Return: A pointer to xsk_buff_pool structure if there is a buffer pool + * attached, configured as zero-copy, and usable by this queue, NULL otherwise. + */ +static struct xsk_buff_pool *ixgbevf_xsk_pool_from_q(struct ixgbevf_ring *ring) +{ + struct ixgbevf_adapter *adapter = ring->q_vector->adapter; + struct xsk_buff_pool *pool = + xsk_get_pool_from_qid(ring->netdev, ring->queue_index); + + if (!rcu_dereference(ring->xdp_prog) && !ring_is_xdp(ring) && + !(adapter->xdp_prog && !adapter->num_xdp_queues)) + return NULL; + + return (pool && pool->dev) ? pool : NULL; +} + /** * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset * @adapter: board private structure @@ -1682,8 +1174,8 @@ static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) * * Configure the Tx descriptor ring after a reset. **/ -static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u64 tdba = ring->dma; @@ -1717,6 +1209,8 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, /* reset ntu and ntc to place SW in sync with hardwdare */ ring->next_to_clean = 0; ring->next_to_use = 0; + ring->pending = 0; + ring->cached_ntu = 0; /* In order to avoid issues WTHRESH + PTHRESH should always be equal * to or less than the number of on chip descriptors, which is @@ -1729,8 +1223,20 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 32; /* PTHRESH = 32 */ /* reinitialize tx_buffer_info */ - memset(ring->tx_buffer_info, 0, - sizeof(struct ixgbevf_tx_buffer) * ring->count); + memset(ring->xdp_sqes, 0, sizeof(struct libeth_sqe) * ring->count); + if (ring_is_xdp(ring)) + libeth_xdpsq_get(&ring->xdpq_lock, ring->netdev, + num_possible_cpus() > adapter->num_xdp_queues); + + ring->xsk_pool = ixgbevf_xsk_pool_from_q(ring); + if (adapter->xdp_prog && + (!adapter->num_xdp_queues || ring_is_xdp(ring)) && ring->xsk_pool) + set_ring_xsk(ring); + else + clear_ring_xsk(ring); + + ring->thresh = ring_is_xsk(ring) ? IXGBEVF_XSK_TX_CLEAN_THRESH(ring) : + XDP_BULK_QUEUE_SIZE; clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); @@ -1766,7 +1272,8 @@ static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring, int index) + struct ixgbevf_ring *ring, int index, + bool rlpml_valid) { struct ixgbe_hw *hw = &adapter->hw; u32 srrctl; @@ -1774,10 +1281,11 @@ static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, srrctl = IXGBE_SRRCTL_DROP_EN; srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; - if (ring_uses_large_buffer(ring)) - srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; + if (rlpml_valid) + srrctl |= DIV_ROUND_UP(ring->rx_buf_len, + IXGBE_SRRCTL_BSIZEPKT_STEP); else - srrctl |= IXGBEVF_RXBUFFER_2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; + srrctl |= ring->rx_buf_len / IXGBE_SRRCTL_BSIZEPKT_STEP; srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); @@ -1799,8 +1307,8 @@ static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) } #define IXGBEVF_MAX_RX_DESC_POLL 10 -static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; @@ -1824,10 +1332,15 @@ static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, if (!wait_loop) pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", reg_idx); + + /* Specification calls for 100 usec of delay after + * RXDCTL.ENABLE is cleared + */ + usleep_range(100, 200); } -static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; @@ -1903,14 +1416,163 @@ static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); } -static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) +{ + struct libeth_fq fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + }; + + if (!fq.pp && !rx_ring->xsk_fqes) + return; + + if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) { + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + } + + if (test_and_clear_bit(__IXGBEVF_RXTX_XSK_RING, &rx_ring->state)) { + struct libeth_xskfq xskfq = { + .fqes = rx_ring->xsk_fqes, + }; + + libeth_xskfq_destroy(&xskfq); + rx_ring->xsk_fqes = NULL; + rx_ring->pending = xskfq.pending; + rx_ring->thresh = xskfq.thresh; + rx_ring->rx_buf_len = xskfq.buf_len; + rx_ring->xsk_pool = NULL; + + return; + } + + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; + + if (!rx_ring->hdr_pp) + return; + + fq = (struct libeth_fq) { + .pp = rx_ring->hdr_pp, + .fqes = rx_ring->hdr_fqes, + }; + + libeth_rx_fq_destroy(&fq); + rx_ring->hdr_fqes = NULL; + rx_ring->hdr_pp = NULL; +} + +static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring) +{ + u32 adapter_flags = rx_ring->q_vector->adapter->flags; + struct libeth_fq fq = { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_MTU, + .xdp = !!rx_ring->xdp_prog, + .buf_len = IXGBEVF_RX_PAGE_LEN(rx_ring->xdp_prog ? + LIBETH_XDP_HEADROOM : + LIBETH_SKB_HEADROOM), + }; + struct xsk_buff_pool *pool; + u32 frame_size; + int ret; + + pool = ixgbevf_xsk_pool_from_q(rx_ring); + if (pool) { + u32 frag_sz = xsk_pool_get_rx_frag_step(pool); + struct libeth_xskfq xskfq = { + .nid = numa_node_id(), + .count = rx_ring->count, + .pool = pool, + }; + + ret = libeth_xskfq_create(&xskfq); + if (ret) + return ret; + + rx_ring->xsk_pool = xskfq.pool; + rx_ring->xsk_fqes = xskfq.fqes; + rx_ring->pending = xskfq.count - 1; + rx_ring->thresh = xskfq.thresh; + rx_ring->rx_buf_len = xskfq.buf_len; + set_ring_xsk(rx_ring); + + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, + rx_ring->queue_index, 0, frag_sz); + if (ret) + goto err; + + ret = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + rx_ring->xsk_pool); + if (ret) + goto err; + + return 0; + } + + /* Some HW requires DMA write sizes to be aligned to 1K, + * which warrants fake header split usage, but this is + * not an issue if the frame size is at its maximum of 3K + */ + frame_size = + IXGBEVF_RX_SRRCTL_BUF_SIZE(READ_ONCE(rx_ring->netdev->mtu)); + fq.hsplit = (adapter_flags & IXGBEVF_FLAG_HSPLIT) && + frame_size < fq.buf_len; + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + return ret; + + rx_ring->pp = fq.pp; + rx_ring->rx_fqes = fq.fqes; + rx_ring->truesize = fq.truesize; + rx_ring->rx_buf_len = fq.buf_len; + + /* XDP RX-queue info */ + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, + rx_ring->queue_index, 0, rx_ring->truesize); + if (ret) + goto err; + + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); + + if (!fq.hsplit) + return 0; + + fq = (struct libeth_fq) { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_HDR, + .xdp = !!rx_ring->xdp_prog, + }; + + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + goto err; + + rx_ring->hdr_pp = fq.pp; + rx_ring->hdr_fqes = fq.fqes; + rx_ring->hdr_truesize = fq.truesize; + + return 0; + +err: + ixgbevf_rx_destroy_pp(rx_ring); + return ret; +} + +void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; union ixgbe_adv_rx_desc *rx_desc; + u8 reg_idx = ring->reg_idx; + bool rlpml_valid = false; u64 rdba = ring->dma; u32 rxdctl; - u8 reg_idx = ring->reg_idx; + int err; /* disable queue to avoid issues while updating state */ rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); @@ -1936,10 +1598,6 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); - /* initialize rx_buffer_info */ - memset(ring->rx_buffer_info, 0, - sizeof(struct ixgbevf_rx_buffer) * ring->count); - /* initialize Rx descriptor 0 */ rx_desc = IXGBEVF_RX_DESC(ring, 0); rx_desc->wb.upper.length = 0; @@ -1947,53 +1605,40 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, /* reset ntu and ntc to place SW in sync with hardwdare */ ring->next_to_clean = 0; ring->next_to_use = 0; - ring->next_to_alloc = 0; + ring->pending = ixgbevf_desc_unused(ring); - ixgbevf_configure_srrctl(adapter, ring, reg_idx); + err = ixgbevf_rx_create_pp(ring); + if (err) { + netdev_err(ring->netdev, + "Failed to create Page Pool for buffer allocation: (%pe), RxQ %d is disabled, driver reload may be needed\n", + ERR_PTR(err), ring->queue_index); + return; + } /* RXDCTL.RLPML does not work on 82599 */ if (adapter->hw.mac.type != ixgbe_mac_82599_vf) { - rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | - IXGBE_RXDCTL_RLPML_EN); - -#if (PAGE_SIZE < 8192) - /* Limit the maximum frame size so we don't overrun the skb */ - if (ring_uses_build_skb(ring) && - !ring_uses_large_buffer(ring)) - rxdctl |= IXGBEVF_MAX_FRAME_BUILD_SKB | - IXGBE_RXDCTL_RLPML_EN; -#endif + u32 pkt_len = + READ_ONCE(adapter->netdev->mtu) + LIBETH_RX_LL_LEN; + + rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); + if (pkt_len <= IXGBE_RXDCTL_RLPMLMASK) { + rxdctl |= pkt_len | IXGBE_RXDCTL_RLPML_EN; + if (pkt_len <= ring->rx_buf_len) + rlpml_valid = true; + } } + ixgbevf_configure_srrctl(adapter, ring, reg_idx, rlpml_valid); + rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); ixgbevf_rx_desc_queue_enable(adapter, ring); - ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); -} - -static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *rx_ring) -{ - struct net_device *netdev = adapter->netdev; - unsigned int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; - /* set build_skb and buffer size flags */ - clear_ring_build_skb_enabled(rx_ring); - clear_ring_uses_large_buffer(rx_ring); - - if (adapter->flags & IXGBEVF_FLAGS_LEGACY_RX) - return; - - if (PAGE_SIZE < 8192) - if (max_frame > IXGBEVF_MAX_FRAME_BUILD_SKB) - set_ring_uses_large_buffer(rx_ring); - - /* 82599 can't rely on RXDCTL.RLPML to restrict the size of the frame */ - if (adapter->hw.mac.type == ixgbe_mac_82599_vf && !ring_uses_large_buffer(rx_ring)) - return; - - set_ring_build_skb_enabled(rx_ring); + if (ring_is_xsk(ring)) + ixgbevf_xsk_alloc_rx_bufs(ring, ring->pending); + else + ixgbevf_alloc_rx_buffers(ring, ring->pending); } /** @@ -2026,7 +1671,6 @@ static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) for (i = 0; i < adapter->num_rx_queues; i++) { struct ixgbevf_ring *rx_ring = adapter->rx_ring[i]; - ixgbevf_set_rx_buffer_len(adapter, rx_ring); ixgbevf_configure_rx_ring(adapter, rx_ring); } } @@ -2177,47 +1821,31 @@ static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) } } -static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) +static void ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) { - struct ixgbe_hw *hw = &adapter->hw; - unsigned int def_q = 0; - unsigned int num_tcs = 0; unsigned int num_rx_queues = adapter->num_rx_queues; unsigned int num_tx_queues = adapter->num_tx_queues; - int err; - - spin_lock_bh(&adapter->mbx_lock); - - /* fetch queue configuration from the PF */ - err = ixgbevf_get_queues(hw, &num_tcs, &def_q); - - spin_unlock_bh(&adapter->mbx_lock); - if (err) - return err; - - if (num_tcs > 1) { + if (adapter->q_caps.num_tcs > 1) { /* we need only one Tx queue */ num_tx_queues = 1; /* update default Tx ring register index */ - adapter->tx_ring[0]->reg_idx = def_q; + adapter->tx_ring[0]->reg_idx = adapter->q_caps.def_tc; /* we need as many queues as traffic classes */ - num_rx_queues = num_tcs; + num_rx_queues = adapter->q_caps.num_tcs; } /* if we have a bad config abort request queue reset */ if ((adapter->num_rx_queues != num_rx_queues) || (adapter->num_tx_queues != num_tx_queues)) { /* force mailbox timeout to prevent further messages */ - hw->mbx.timeout = 0; + adapter->hw.mbx.timeout = 0; /* wait for watchdog to come around and bail us out */ set_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, &adapter->state); } - - return 0; } static void ixgbevf_configure(struct ixgbevf_adapter *adapter) @@ -2383,106 +2011,46 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter) * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue * @rx_ring: ring to free buffers from **/ -static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) +void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) { - u16 i = rx_ring->next_to_clean; + if (ring_is_xsk(rx_ring)) { + ixgbevf_rx_xsk_ring_free_buffs(rx_ring); + goto reset; + } /* Free Rx ring sk_buff */ - if (rx_ring->skb) { - dev_kfree_skb(rx_ring->skb); - rx_ring->skb = NULL; - } + libeth_xdp_return_stash(&rx_ring->xdp_stash); /* Free all the Rx ring pages */ - while (i != rx_ring->next_to_alloc) { - struct ixgbevf_rx_buffer *rx_buffer; - - rx_buffer = &rx_ring->rx_buffer_info[i]; - - /* Invalidate cache lines that may have been written to by - * device so that we avoid corrupting memory. - */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - ixgbevf_rx_bufsz(rx_ring), - DMA_FROM_DEVICE); - - /* free resources associated with mapping */ - dma_unmap_page_attrs(rx_ring->dev, - rx_buffer->dma, - ixgbevf_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBEVF_RX_DMA_ATTR); - - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); - - i++; - if (i == rx_ring->count) + for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { + const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; + const struct libeth_fqe *hdr_fqe = rx_ring->hdr_fqes ? + &rx_ring->hdr_fqes[i] : + NULL; + + libeth_rx_recycle_slow(rx_fqe->netmem); + if (hdr_fqe) + libeth_rx_recycle_slow(hdr_fqe->netmem); + if (unlikely(++i == rx_ring->count)) i = 0; } - rx_ring->next_to_alloc = 0; +reset: rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; + rx_ring->pending = 0; } /** * ixgbevf_clean_tx_ring - Free Tx Buffers * @tx_ring: ring to be cleaned **/ -static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) +void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) { - u16 i = tx_ring->next_to_clean; - struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; - - while (i != tx_ring->next_to_use) { - union ixgbe_adv_tx_desc *eop_desc, *tx_desc; - - /* Free all the Tx ring sk_buffs */ - if (ring_is_xdp(tx_ring)) - page_frag_free(tx_buffer->data); - else - dev_kfree_skb_any(tx_buffer->skb); - - /* unmap skb header data */ - dma_unmap_single(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + struct libeth_sq_napi_stats stats = { }; - /* check for eop_desc to determine the end of the packet */ - eop_desc = tx_buffer->next_to_watch; - tx_desc = IXGBEVF_TX_DESC(tx_ring, i); - - /* unmap remaining buffers */ - while (tx_desc != eop_desc) { - tx_buffer++; - tx_desc++; - i++; - if (unlikely(i == tx_ring->count)) { - i = 0; - tx_buffer = tx_ring->tx_buffer_info; - tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); - } - - /* unmap any remaining paged data */ - if (dma_unmap_len(tx_buffer, len)) - dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); - } - - /* move us one more past the eop_desc for start of next pkt */ - tx_buffer++; - i++; - if (unlikely(i == tx_ring->count)) { - i = 0; - tx_buffer = tx_ring->tx_buffer_info; - } - } + ixgbevf_clean_tx_num(tx_ring, false, ixgbevf_desc_used(tx_ring), + &stats, NULL); /* reset next_to_use and next_to_clean */ tx_ring->next_to_use = 0; @@ -2498,8 +2066,10 @@ static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) { int i; - for (i = 0; i < adapter->num_rx_queues; i++) + for (i = 0; i < adapter->num_rx_queues; i++) { ixgbevf_clean_rx_ring(adapter->rx_ring[i]); + ixgbevf_rx_destroy_pp(adapter->rx_ring[i]); + } } /** @@ -2513,13 +2083,20 @@ static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) for (i = 0; i < adapter->num_tx_queues; i++) ixgbevf_clean_tx_ring(adapter->tx_ring[i]); for (i = 0; i < adapter->num_xdp_queues; i++) - ixgbevf_clean_tx_ring(adapter->xdp_ring[i]); + ixgbevf_clean_xdp_ring(adapter->xdp_ring[i]); +} + +void ixgbevf_flush_tx_queue(struct ixgbevf_ring *ring) +{ + u8 reg_idx = ring->reg_idx; + + IXGBE_WRITE_REG(&ring->q_vector->adapter->hw, IXGBE_VFTXDCTL(reg_idx), + IXGBE_TXDCTL_SWFLSH); } void ixgbevf_down(struct ixgbevf_adapter *adapter) { struct net_device *netdev = adapter->netdev; - struct ixgbe_hw *hw = &adapter->hw; int i; /* signal that we are down to the interrupt handler */ @@ -2545,19 +2122,11 @@ void ixgbevf_down(struct ixgbevf_adapter *adapter) timer_delete_sync(&adapter->service_timer); /* disable transmits in the hardware now that interrupts are off */ - for (i = 0; i < adapter->num_tx_queues; i++) { - u8 reg_idx = adapter->tx_ring[i]->reg_idx; - - IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), - IXGBE_TXDCTL_SWFLSH); - } - - for (i = 0; i < adapter->num_xdp_queues; i++) { - u8 reg_idx = adapter->xdp_ring[i]->reg_idx; + for (i = 0; i < adapter->num_tx_queues; i++) + ixgbevf_flush_tx_queue(adapter->tx_ring[i]); - IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), - IXGBE_TXDCTL_SWFLSH); - } + for (i = 0; i < adapter->num_xdp_queues; i++) + ixgbevf_flush_tx_queue(adapter->xdp_ring[i]); if (!pci_channel_offline(adapter->pdev)) ixgbevf_reset(adapter); @@ -2634,6 +2203,56 @@ static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, return 0; } +static void ixgbevf_cfg_queue_caps(struct ixgbevf_adapter *adapter) +{ + struct ixgbe_hw *hw = &adapter->hw; + u32 max_qpairs, rss; + int err; + + adapter->q_caps = (typeof(adapter->q_caps)) { + .min_rxqs = 1, + .min_txqs = 1, + .max_rxqs = 1, + .max_txqs = 1, + }; + + /* fetch queue configuration from the PF */ + spin_lock_bh(&adapter->mbx_lock); + err = ixgbevf_get_queues(hw, &adapter->q_caps.num_tcs, + &adapter->q_caps.def_tc); + spin_unlock_bh(&adapter->mbx_lock); + if (err) + return; + + /* we need as many queues as traffic classes */ + if (adapter->q_caps.num_tcs > 1) { + adapter->q_caps.min_rxqs = adapter->q_caps.num_tcs; + adapter->q_caps.max_rxqs = adapter->q_caps.num_tcs; + return; + } + + switch (hw->api_version) { + case ixgbe_mbox_api_11: + case ixgbe_mbox_api_12: + case ixgbe_mbox_api_13: + case ixgbe_mbox_api_14: + case ixgbe_mbox_api_15: + case ixgbe_mbox_api_16: + case ixgbe_mbox_api_17: + max_qpairs = min_t(u32, hw->mac.max_rx_queues, + hw->mac.max_tx_queues); + break; + default: + max_qpairs = IXGBEVF_MAX_RSS_QUEUES; + break; + } + + rss = min_t(u32, max_qpairs, num_online_cpus()); + + adapter->q_caps.max_rxqs = rss; + adapter->q_caps.max_txqs = rss; +} + /** * ixgbevf_set_num_queues - Allocate queues for device, feature dependent * @adapter: board private structure to initialize @@ -2647,51 +2266,29 @@ static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, **/ static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) { - struct ixgbe_hw *hw = &adapter->hw; - unsigned int def_q = 0; - unsigned int num_tcs = 0; - int err; - - /* Start with base case */ - adapter->num_rx_queues = 1; - adapter->num_tx_queues = 1; - adapter->num_xdp_queues = 0; - - spin_lock_bh(&adapter->mbx_lock); - - /* fetch queue configuration from the PF */ - err = ixgbevf_get_queues(hw, &num_tcs, &def_q); + ixgbevf_cfg_queue_caps(adapter); - spin_unlock_bh(&adapter->mbx_lock); + if (adapter->q_caps.num_tcs > 1) { + adapter->num_rx_queues = adapter->q_caps.max_rxqs; + adapter->num_tx_queues = adapter->q_caps.max_txqs; + adapter->num_xdp_queues = 0; - if (err) return; - - /* we need as many queues as traffic classes */ - if (num_tcs > 1) { - adapter->num_rx_queues = num_tcs; } else { - u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES); - - switch (hw->api_version) { - case ixgbe_mbox_api_11: - case ixgbe_mbox_api_12: - case ixgbe_mbox_api_13: - case ixgbe_mbox_api_14: - case ixgbe_mbox_api_15: - case ixgbe_mbox_api_16: - case ixgbe_mbox_api_17: - if (adapter->xdp_prog && - hw->mac.max_tx_queues == rss) - rss = rss > 3 ? 2 : 1; - - adapter->num_rx_queues = rss; - adapter->num_tx_queues = rss; - adapter->num_xdp_queues = adapter->xdp_prog ? rss : 0; - break; - default: - break; - } + u32 max_qpairs = min_t(u32, adapter->q_caps.max_rxqs, + adapter->q_caps.max_txqs); + + adapter->num_req_qpairs = + adapter->num_req_qpairs ? : adapter->q_caps.max_rxqs; + WARN_ON_ONCE(adapter->num_req_qpairs > max_qpairs); + adapter->num_rx_queues = + min_t(u32, adapter->num_req_qpairs, max_qpairs); + adapter->num_tx_queues = adapter->num_rx_queues; + // if (adapter->num_rx_queues + adapter->num_tx_queues <= + // adapter->q_caps.max_txqs && adapter->xdp_prog) + // adapter->num_xdp_queues = adapter->num_rx_queues; + // else + adapter->num_xdp_queues = 0; } } @@ -3003,7 +2600,7 @@ static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) * @adapter: board private structure to initialize * **/ -static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) +int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) { int err; @@ -3044,7 +2641,7 @@ static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) * We go through and clear interrupt specific resources and reset the structure * to pre-load conditions **/ -static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) +void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) { adapter->num_tx_queues = 0; adapter->num_xdp_queues = 0; @@ -3084,6 +2681,9 @@ static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) goto out; } + if (adapter->hw.mac.type == ixgbe_mac_82599_vf) + adapter->flags |= IXGBEVF_FLAG_HSPLIT; + /* assume legacy case in which PF would only give VF 2 queues */ hw->mac.max_tx_queues = 2; hw->mac.max_rx_queues = 2; @@ -3259,8 +2859,6 @@ static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter) if (netif_carrier_ok(adapter->netdev)) { for (i = 0; i < adapter->num_tx_queues; i++) set_check_for_tx_hang(adapter->tx_ring[i]); - for (i = 0; i < adapter->num_xdp_queues; i++) - set_check_for_tx_hang(adapter->xdp_ring[i]); } /* get one bit for every active Tx/Rx interrupt vector */ @@ -3404,16 +3002,19 @@ static void ixgbevf_service_task(struct work_struct *work) **/ void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) { - ixgbevf_clean_tx_ring(tx_ring); + if (!ring_is_xdp(tx_ring)) + ixgbevf_clean_tx_ring(tx_ring); + else + ixgbevf_clean_xdp_ring(tx_ring); - vfree(tx_ring->tx_buffer_info); - tx_ring->tx_buffer_info = NULL; + vfree(tx_ring->tx_sqes); + tx_ring->tx_sqes = NULL; /* if not set, then don't free */ if (!tx_ring->desc) return; - dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, + dma_free_coherent(tx_ring->dev, tx_ring->dma_size, tx_ring->desc, tx_ring->dma); tx_ring->desc = NULL; @@ -3448,18 +3049,19 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); int size; - size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; - tx_ring->tx_buffer_info = vmalloc(size); - if (!tx_ring->tx_buffer_info) + size = sizeof(struct libeth_sqe) * tx_ring->count; + + tx_ring->tx_sqes = vmalloc(size); + if (!tx_ring->tx_sqes) goto err; u64_stats_init(&tx_ring->syncp); /* round up to nearest 4K */ - tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); - tx_ring->size = ALIGN(tx_ring->size, 4096); + tx_ring->dma_size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); + tx_ring->dma_size = ALIGN(tx_ring->dma_size, 4096); - tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, + tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->dma_size, &tx_ring->dma, GFP_KERNEL); if (!tx_ring->desc) goto err; @@ -3467,8 +3069,8 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) return 0; err: - vfree(tx_ring->tx_buffer_info); - tx_ring->tx_buffer_info = NULL; + vfree(tx_ring->tx_sqes); + tx_ring->tx_sqes = NULL; hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit descriptor ring\n"); return -ENOMEM; } @@ -3514,48 +3116,40 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) return err; } +static struct device *ixgbevf_dma_dev_from_ring(struct ixgbevf_ring *ring) +{ + return &ring->q_vector->adapter->pdev->dev; +} + /** - * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) + * ixgbevf_setup_rx_resources - allocate Rx resources * @adapter: board private structure * @rx_ring: Rx descriptor ring (for a specific queue) to setup * - * Returns 0 on success, negative on failure + * Returns: 0 on success, negative on failure. **/ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring) { - int size; - - size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; - rx_ring->rx_buffer_info = vmalloc(size); - if (!rx_ring->rx_buffer_info) - goto err; - u64_stats_init(&rx_ring->syncp); /* Round up to nearest 4K */ - rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); - rx_ring->size = ALIGN(rx_ring->size, 4096); + rx_ring->dma_size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); + rx_ring->dma_size = ALIGN(rx_ring->dma_size, 4096); - rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, + rx_ring->desc = dma_alloc_coherent(ixgbevf_dma_dev_from_ring(rx_ring), + rx_ring->dma_size, &rx_ring->dma, GFP_KERNEL); - if (!rx_ring->desc) - goto err; - - /* XDP RX-queue info */ - if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index, 0) < 0) - goto err; + if (!rx_ring->desc) { + dev_err(rx_ring->dev, + "Unable to allocate memory for the Rx descriptor ring\n"); + return -ENOMEM; + } - rx_ring->xdp_prog = adapter->xdp_prog; + rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; -err: - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; - dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); - return -ENOMEM; } /** @@ -3597,15 +3191,12 @@ static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) { ixgbevf_clean_rx_ring(rx_ring); + ixgbevf_rx_destroy_pp(rx_ring); + rcu_assign_pointer(rx_ring->xdp_prog, NULL); - rx_ring->xdp_prog = NULL; - xdp_rxq_info_unreg(&rx_ring->xdp_rxq); - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; - - dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, + dma_free_coherent(ixgbevf_dma_dev_from_ring(rx_ring), + rx_ring->dma_size, rx_ring->desc, rx_ring->dma); - rx_ring->desc = NULL; } @@ -3799,11 +3390,10 @@ static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); } -static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, - u8 *hdr_len, - struct ixgbevf_ipsec_tx_data *itd) +static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, struct libeth_sqe *first, + u8 *hdr_len, struct ixgbevf_ipsec_tx_data *itd) { + struct ixgbevf_skb_sqe_priv *priv = (void *)&first->priv; u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; struct sk_buff *skb = first->skb; union { @@ -3829,7 +3419,7 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, if (err < 0) return err; - if (eth_p_mpls(first->protocol)) + if (eth_p_mpls(priv->protocol)) ip.hdr = skb_inner_network_header(skb); else ip.hdr = skb_network_header(skb); @@ -3854,12 +3444,12 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; ip.v4->tot_len = 0; - first->tx_flags |= IXGBE_TX_FLAGS_TSO | + priv->tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM | IXGBE_TX_FLAGS_IPV4; } else { ip.v6->payload_len = 0; - first->tx_flags |= IXGBE_TX_FLAGS_TSO | + priv->tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM; } @@ -3874,8 +3464,8 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen)); /* update gso size and bytecount with header size */ - first->gso_segs = skb_shinfo(skb)->gso_segs; - first->bytecount += (first->gso_segs - 1) * *hdr_len; + first->packets = skb_shinfo(skb)->gso_segs; + first->bytes += (first->packets - 1) * *hdr_len; /* mss_l4len_id: use 1 as index for TSO */ mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT; @@ -3888,7 +3478,7 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ vlan_macip_lens = l4.hdr - ip.hdr; vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT; - vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; + vlan_macip_lens |= priv->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd, mss_l4len_idx); @@ -3897,9 +3487,10 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, } static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libeth_sqe *first, struct ixgbevf_ipsec_tx_data *itd) { + struct ixgbevf_skb_sqe_priv *priv = (void *)&first->priv; struct sk_buff *skb = first->skb; u32 vlan_macip_lens = 0; u32 fceof_saidx = 0; @@ -3926,17 +3517,17 @@ static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, goto no_csum; } - if (first->protocol == htons(ETH_P_IP)) + if (priv->protocol == htons(ETH_P_IP)) type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; /* update TX checksum flag */ - first->tx_flags |= IXGBE_TX_FLAGS_CSUM; + priv->tx_flags |= IXGBE_TX_FLAGS_CSUM; vlan_macip_lens = skb_checksum_start_offset(skb) - skb_network_offset(skb); no_csum: /* vlan_macip_lens: MACLEN, VLAN tag */ vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; - vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; + vlan_macip_lens |= priv->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; fceof_saidx |= itd->pfsa; type_tucmd |= itd->flags | itd->trailer_len; @@ -3993,20 +3584,21 @@ static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, } static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libeth_sqe *first, const u8 hdr_len) { + struct ixgbevf_skb_sqe_priv *priv = (void *)&first->priv; struct sk_buff *skb = first->skb; - struct ixgbevf_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; + u32 tx_flags = priv->tx_flags; + struct libeth_sqe *used_sqe; skb_frag_t *frag; dma_addr_t dma; unsigned int data_len, size; - u32 tx_flags = first->tx_flags; __le32 cmd_type = ixgbevf_tx_cmd_type(tx_flags); - u16 i = tx_ring->next_to_use; + u16 ntu = tx_ring->next_to_use; - tx_desc = IXGBEVF_TX_DESC(tx_ring, i); + tx_desc = IXGBEVF_TX_DESC(tx_ring, ntu); ixgbevf_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len); @@ -4015,15 +3607,16 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); - tx_buffer = first; - for (frag = &skb_shinfo(skb)->frags[0];; frag++) { - if (dma_mapping_error(tx_ring->dev, dma)) + + if (unlikely(dma_mapping_error(tx_ring->dev, dma))) goto dma_error; /* record length, and DMA address */ - dma_unmap_len_set(tx_buffer, len, size); - dma_unmap_addr_set(tx_buffer, dma, dma); + used_sqe = &tx_ring->tx_sqes[ntu]; + dma_unmap_len_set(used_sqe, len, size); + dma_unmap_addr_set(used_sqe, dma, dma); + used_sqe->type = LIBETH_SQE_FRAG; tx_desc->read.buffer_addr = cpu_to_le64(dma); @@ -4031,11 +3624,11 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); - i++; + ntu++; tx_desc++; - if (i == tx_ring->count) { + if (ntu == tx_ring->count) { tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); - i = 0; + ntu = 0; } tx_desc->read.olinfo_status = 0; @@ -4050,11 +3643,11 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); - i++; + ntu++; tx_desc++; - if (i == tx_ring->count) { + if (ntu == tx_ring->count) { tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); - i = 0; + ntu = 0; } tx_desc->read.olinfo_status = 0; @@ -4063,18 +3656,14 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, DMA_TO_DEVICE); - - tx_buffer = &tx_ring->tx_buffer_info[i]; } /* write last descriptor with RS and EOP bits */ cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); tx_desc->read.cmd_type_len = cmd_type; - /* set the timestamp */ - first->time_stamp = jiffies; - skb_tx_timestamp(skb); + first->type = LIBETH_SQE_SKB; /* Force memory writes to complete before letting h/w know there * are new descriptors to fetch. (Only applicable for weak-ordered @@ -4086,47 +3675,48 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, wmb(); /* set next_to_watch value indicating a packet is present */ - first->next_to_watch = tx_desc; + tx_ring->xdp_sqes[tx_ring->cached_ntu].rs_idx = ntu + 1; - i++; - if (i == tx_ring->count) - i = 0; + ntu++; + if (ntu == tx_ring->count) + ntu = 0; - tx_ring->next_to_use = i; + tx_ring->next_to_use = ntu; + tx_ring->cached_ntu = ntu; /* notify HW of packet */ - ixgbevf_write_tail(tx_ring, i); + ixgbevf_write_tail(tx_ring, ntu); return; dma_error: dev_err(tx_ring->dev, "TX DMA map failed\n"); - tx_buffer = &tx_ring->tx_buffer_info[i]; + used_sqe = &tx_ring->tx_sqes[ntu]; /* clear dma mappings for failed tx_buffer_info map */ - while (tx_buffer != first) { - if (dma_unmap_len(tx_buffer, len)) + while (used_sqe != first) { + if (dma_unmap_len(used_sqe, len)) dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), + dma_unmap_addr(used_sqe, dma), + dma_unmap_len(used_sqe, len), DMA_TO_DEVICE); - dma_unmap_len_set(tx_buffer, len, 0); + dma_unmap_len_set(used_sqe, len, 0); - if (i-- == 0) - i += tx_ring->count; - tx_buffer = &tx_ring->tx_buffer_info[i]; + if (ntu-- == 0) + ntu += tx_ring->count; + used_sqe = &tx_ring->tx_sqes[ntu]; } - if (dma_unmap_len(tx_buffer, len)) + if (dma_unmap_len(used_sqe, len)) dma_unmap_single(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), + dma_unmap_addr(used_sqe, dma), + dma_unmap_len(used_sqe, len), DMA_TO_DEVICE); - dma_unmap_len_set(tx_buffer, len, 0); + dma_unmap_len_set(used_sqe, len, 0); - dev_kfree_skb_any(tx_buffer->skb); - tx_buffer->skb = NULL; + dev_kfree_skb_any(used_sqe->skb); + used_sqe->skb = NULL; - tx_ring->next_to_use = i; + tx_ring->next_to_use = ntu; } static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) @@ -4161,7 +3751,8 @@ static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) static int ixgbevf_xmit_frame_ring(struct sk_buff *skb, struct ixgbevf_ring *tx_ring) { - struct ixgbevf_tx_buffer *first; + struct ixgbevf_skb_sqe_priv *priv; + struct libeth_sqe *first; int tso; u32 tx_flags = 0; u16 count = TXD_USE_COUNT(skb_headlen(skb)); @@ -4198,10 +3789,11 @@ static int ixgbevf_xmit_frame_ring(struct sk_buff *skb, } /* record the location of the first descriptor for this packet */ - first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; + first = &tx_ring->tx_sqes[tx_ring->next_to_use]; + priv = (void *)&first->priv; first->skb = skb; - first->bytecount = skb->len; - first->gso_segs = 1; + first->bytes = skb->len; + first->packets = 1; if (skb_vlan_tag_present(skb)) { tx_flags |= skb_vlan_tag_get(skb); @@ -4210,8 +3802,8 @@ static int ixgbevf_xmit_frame_ring(struct sk_buff *skb, } /* record initial flags and protocol */ - first->tx_flags = tx_flags; - first->protocol = vlan_get_protocol(skb); + priv->tx_flags = tx_flags; + priv->protocol = vlan_get_protocol(skb); #ifdef CONFIG_IXGBEVF_IPSEC if (xfrm_offload(skb) && !ixgbevf_ipsec_tx(tx_ring, first, &ipsec_tx)) @@ -4292,6 +3884,18 @@ static int ixgbevf_set_mac(struct net_device *netdev, void *p) return 0; } +static bool ixgbevf_xdp_mtu_ok(const struct ixgbevf_adapter *adapter, + const struct bpf_prog *prog, unsigned int mtu) +{ + u32 frame_size = mtu + LIBETH_RX_LL_LEN; + bool requires_mbuf; + + requires_mbuf = frame_size > IXGBEVF_RX_PAGE_LEN(LIBETH_XDP_HEADROOM) || + adapter->flags & IXGBEVF_FLAG_HSPLIT; + + return prog->aux->xdp_has_frags || !requires_mbuf; +} + /** * ixgbevf_change_mtu - Change the Maximum Transfer Unit * @netdev: network interface device structure @@ -4307,8 +3911,10 @@ static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) int ret; /* prevent MTU being changed to a size unsupported by XDP */ - if (adapter->xdp_prog) { - dev_warn(&adapter->pdev->dev, "MTU cannot be changed while XDP program is loaded\n"); + if (adapter->xdp_prog && + !ixgbevf_xdp_mtu_ok(adapter, adapter->xdp_prog, new_mtu)) { + netdev_warn(netdev, + "MTU value provided cannot be set while current XDP program is attached\n"); return -EPERM; } @@ -4468,24 +4074,24 @@ ixgbevf_features_check(struct sk_buff *skb, struct net_device *dev, return features; } -static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) +static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) { - int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; struct ixgbevf_adapter *adapter = netdev_priv(dev); struct bpf_prog *old_prog; - /* verify ixgbevf ring attributes are sufficient for XDP */ - for (i = 0; i < adapter->num_rx_queues; i++) { - struct ixgbevf_ring *ring = adapter->rx_ring[i]; - - if (frame_size > ixgbevf_rx_bufsz(ring)) - return -EINVAL; + if (prog && !ixgbevf_xdp_mtu_ok(adapter, prog, READ_ONCE(dev->mtu))) { + NL_SET_ERR_MSG_MOD(extack, + "Configured MTU or HW limitations require non-linear frames and XDP prog does not support frags"); + return -EOPNOTSUPP; } old_prog = xchg(&adapter->xdp_prog, prog); /* If transitioning XDP modes reconfigure rings */ if (!!prog != !!old_prog) { + xdp_features_clear_redirect_target(dev); + /* Hardware has to reinitialize queues and interrupts to * match packet buffer alignment. Unfortunately, the * hardware is not flexible enough to do this dynamically. @@ -4499,10 +4105,15 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (netif_running(dev)) ixgbevf_open(dev); } else { - for (i = 0; i < adapter->num_rx_queues; i++) - xchg(&adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); + for (int i = 0; i < adapter->num_rx_queues; i++) + rcu_assign_pointer(adapter->rx_ring[i]->xdp_prog, + adapter->xdp_prog); + synchronize_net(); } + if (prog) + xdp_features_set_redirect_target(dev, true); + if (old_prog) bpf_prog_put(old_prog); @@ -4513,7 +4124,10 @@ static int ixgbevf_xdp(struct net_device *dev, struct netdev_bpf *xdp) { switch (xdp->command) { case XDP_SETUP_PROG: - return ixgbevf_xdp_setup(dev, xdp->prog); + return ixgbevf_xdp_setup(dev, xdp->prog, xdp->extack); + case XDP_SETUP_XSK_POOL: + return ixgbevf_setup_xsk_pool(netdev_priv(dev), xdp->xsk.pool, + xdp->xsk.queue_id); default: return -EINVAL; } @@ -4533,6 +4147,8 @@ static const struct net_device_ops ixgbevf_netdev_ops = { .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, .ndo_features_check = ixgbevf_features_check, .ndo_bpf = ixgbevf_xdp, + .ndo_xdp_xmit = ixgbevf_xdp_xmit, + .ndo_xsk_wakeup = ixgbevf_xsk_wakeup, }; static void ixgbevf_assign_netdev_ops(struct net_device *dev) @@ -4665,7 +4281,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; - netdev->xdp_features = NETDEV_XDP_ACT_BASIC; + libeth_xdp_set_features_noredir(netdev, NULL, IXGBEVF_XSK_MAX_ZC_FRAGS); /* MTU range: 68 - 1504 or 9710 */ netdev->min_mtu = ETH_MIN_MTU; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h new file mode 100644 index 00000000000000..ac36fac9f79717 --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2010-2026 Intel Corporation */ + +#ifndef _IXGBEVF_TXRX_LIB_H_ +#define _IXGBEVF_TXRX_LIB_H_ + +#include "ixgbevf.h" + +/** + * ixgbevf_is_non_eop - process handling of non-EOP buffers + * @rx_ring: Rx ring being processed + * @rx_desc: Rx descriptor for current buffer + * + * This function updates next to clean. If the buffer is an EOP buffer + * this function exits returning false, otherwise it will place the + * sk_buff in the next buffer to be chained and return true indicating + * that this is in fact a non-EOP buffer. + **/ +static inline bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, + union ixgbe_adv_rx_desc *rx_desc) +{ + u32 ntc = rx_ring->next_to_clean + 1; + + /* fetch, update, and store next to clean */ + ntc = (ntc < rx_ring->count) ? ntc : 0; + rx_ring->next_to_clean = ntc; + rx_ring->pending++; + + prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); + + if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) + return false; + + return true; +} + +/** + * ixgbevf_cleanup_headers - Correct corrupted or empty headers + * @rx_ring: rx descriptor ring packet is being transacted on + * @rx_desc: pointer to the EOP Rx descriptor + * @skb: pointer to current skb being fixed + * + * Check for corrupted packet headers caused by senders on the local L2 + * embedded NIC switch not setting up their Tx Descriptors right. These + * should be very rare. + * + * Also address the case where we are pulling data in on pages only + * and as such no data is present in the skb header. + * + * In addition if skb is not at least 60 bytes we need to pad it so that + * it is large enough to qualify as a valid Ethernet frame. + * + * Returns true if an error was encountered and skb was freed. + **/ +static inline bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + /* verify that the packet does not have any known errors */ + if (unlikely(ixgbevf_test_staterr(rx_desc, + IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { + struct net_device *netdev = rx_ring->netdev; + + if (!(netdev->features & NETIF_F_RXALL)) { + dev_kfree_skb_any(skb); + return true; + } + } + + return false; +} + +#define IXGBE_RSS_L4_TYPES_MASK \ + ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ + (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ + (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ + (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) + +static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + u16 rss_type; + + if (!(ring->netdev->features & NETIF_F_RXHASH)) + return; + + rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & + IXGBE_RXDADV_RSSTYPE_MASK; + + if (!rss_type) + return; + + skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), + (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? + PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); +} + +/** + * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum + * @ring: structure containing ring specific data + * @rx_desc: current Rx descriptor being processed + * @skb: skb currently being received and modified + **/ +static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + skb_checksum_none_assert(skb); + + /* Rx csum disabled */ + if (!(ring->netdev->features & NETIF_F_RXCSUM)) + return; + + /* if IP and error */ + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && + ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { + ring->rx_stats.csum_err++; + return; + } + + if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) + return; + + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { + ring->rx_stats.csum_err++; + return; + } + + /* It must be a TCP or UDP packet with a valid checksum */ + skb->ip_summed = CHECKSUM_UNNECESSARY; +} + +/** + * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor + * @rx_ring: rx descriptor ring packet is being transacted on + * @rx_desc: pointer to the EOP Rx descriptor + * @skb: pointer to current skb being populated + * + * This function checks the ring, descriptor, and packet information in + * order to populate the checksum, VLAN, protocol, and other fields within + * the skb. + **/ +static inline void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + ixgbevf_rx_hash(rx_ring, rx_desc, skb); + ixgbevf_rx_checksum(rx_ring, rx_desc, skb); + + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { + u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); + unsigned long *active_vlans = netdev_priv(rx_ring->netdev); + + if (test_bit(vid & VLAN_VID_MASK, active_vlans)) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); + } + + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP)) + ixgbevf_ipsec_rx(rx_ring, rx_desc, skb); +} + +#endif /* _IXGBEVF_TXRX_LIB_H_ */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h new file mode 100644 index 00000000000000..452b0013898918 --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h @@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2010-2026 Intel Corporation */ + +#ifndef _IXGBEVF_XDP_LIB_H_ +#define _IXGBEVF_XDP_LIB_H_ + +#include + +#include "ixgbevf.h" + +static inline u16 ixgbevf_tx_get_num_sent(struct ixgbevf_ring *tx_ring, + u16 budget) +{ + u16 ntc = tx_ring->next_to_clean; + u16 to_clean = 0; + + while (likely(to_clean < budget)) { + u32 idx = tx_ring->xdp_sqes[ntc].rs_idx; + union ixgbe_adv_tx_desc *rs_desc; + + if (!idx--) + break; + + smp_rmb(); + + rs_desc = IXGBEVF_TX_DESC(tx_ring, idx); + + if (!(rs_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) + break; + + tx_ring->xdp_sqes[ntc].rs_idx = 0; + + to_clean += + (idx >= ntc ? idx : idx + tx_ring->count) - ntc + 1; + + ntc = (idx + 1 == tx_ring->count) ? 0 : idx + 1; + } + + return to_clean; +} + +static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring, + bool in_napi, u16 to_clean) +{ + struct libeth_xdpsq_napi_stats stats = { }; + bool xsk_ring = ring_is_xsk(xdp_ring); + u32 ntc = xdp_ring->next_to_clean; + struct xdp_frame_bulk cbulk; + struct libeth_cq_pp cp = { + .bq = &cbulk, + .dev = xdp_ring->dev, + .xss = &stats, + .napi = in_napi, + }; + u32 xsk_frames = 0; + + xdp_frame_bulk_init(&cbulk); + xdp_ring->pending -= to_clean; + + while (likely(to_clean--)) { + xsk_frames += xsk_ring && + likely(!xdp_ring->xdp_sqes[ntc].type) ? 1 : 0; + libeth_xdp_complete_tx(&xdp_ring->xdp_sqes[ntc], &cp); + ntc++; + ntc = unlikely(ntc == xdp_ring->count) ? 0 : ntc; + } + + xdp_ring->next_to_clean = ntc; + xdp_flush_frame_bulk(&cbulk); + if (xsk_frames) + xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); +} + +static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) +{ + struct ixgbevf_ring *xdp_ring = xdpsq; + + libeth_xdpsq_lock(&xdp_ring->xdpq_lock); + if (unlikely(ixgbevf_desc_unused(xdp_ring) < xdp_ring->thresh)) { + u16 to_clean = + ixgbevf_tx_get_num_sent(xdp_ring, xdp_ring->pending); + + if (likely(to_clean)) + ixgbevf_clean_xdp_num(xdp_ring, true, to_clean); + } + + if (unlikely(!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, + &xdp_ring->state))) { + struct ixgbe_adv_tx_context_desc *context_desc; + + set_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &xdp_ring->state); + + context_desc = IXGBEVF_TX_CTXTDESC(xdp_ring, 0); + context_desc->vlan_macip_lens = + cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT); + context_desc->fceof_saidx = 0; + context_desc->type_tucmd_mlhl = + cpu_to_le32(IXGBE_TXD_CMD_DEXT | + IXGBE_ADVTXD_DTYP_CTXT); + context_desc->mss_l4len_idx = 0; + + xdp_ring->next_to_use = 1; + xdp_ring->pending = 1; + xdp_ring->xdp_sqes[0].type = LIBETH_SQE_CTX; + + /* Finish descriptor writes before bumping tail */ + wmb(); + ixgbevf_write_tail(xdp_ring, 1); + } + + *sq = (struct libeth_xdpsq) { + .count = xdp_ring->count, + .descs = xdp_ring->desc, + .lock = &xdp_ring->xdpq_lock, + .ntu = &xdp_ring->next_to_use, + .pending = &xdp_ring->pending, + .pool = xdp_ring->xsk_pool, + .sqes = xdp_ring->xdp_sqes, + }; + + return ixgbevf_desc_unused(xdp_ring); +} + +static inline struct netdev_queue * +ixgbevf_xdp_tx_get_nq(struct ixgbevf_ring *tx_ring) +{ + struct netdev_queue *nq; + + nq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->queue_index); + __netif_tx_lock(nq, smp_processor_id()); + + return nq; +} + +static inline void ixgbevf_xdp_tx_put_nq(struct ixgbevf_ring *tx_ring) +{ + struct netdev_queue *nq; + + nq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->queue_index); + __netif_tx_unlock(nq); +} + +static inline u32 ixgbevf_prep_tx_sq(void *xdpsq, struct libeth_xdpsq *sq) +{ + struct ixgbe_adv_tx_context_desc *context_desc; + struct ixgbevf_ring *tx_ring = xdpsq; + struct netdev_queue *nq; + u32 num_unused, ntu; + + /* Serialization of producers in .ndo_start_xmit(), + * .ndo_xdp_xmit() and XDP_TX + */ + nq = ixgbevf_xdp_tx_get_nq(tx_ring); + + /* We need at least 1 additional descriptor for context */ + num_unused = ixgbevf_desc_unused(tx_ring); + if (num_unused < 2) + return 0; + + /* Inform the stack that queue is transmitting to avoid Tx timeout */ + txq_trans_cond_update(nq); + + /* Shared TxQ cleaning was done beforehand */ + + /* Instead of sending a context descriptor once for an XDP-only ring, + * do this before sending each bulk + */ + ntu = tx_ring->next_to_use; + context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, ntu); + tx_ring->xdp_sqes[ntu].type = LIBETH_SQE_CTX; + context_desc->vlan_macip_lens = + cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT); + context_desc->fceof_saidx = 0; + context_desc->type_tucmd_mlhl = + cpu_to_le32(IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT); + context_desc->mss_l4len_idx = 0; + + ntu++; + num_unused--; + ntu = ntu == tx_ring->count ? 0 : ntu; + tx_ring->next_to_use = ntu; + + *sq = (struct libeth_xdpsq) { + .count = tx_ring->count, + .descs = tx_ring->desc, + .lock = &tx_ring->xdpq_lock, + .ntu = &tx_ring->next_to_use, + /* This value is ignored in shared queues */ + .pending = &tx_ring->pending, + .pool = tx_ring->xsk_pool, + .sqes = tx_ring->xdp_sqes, + }; + + return num_unused; +} + +static inline void +ixgbevf_xdp_tx_unprep(void *xdpsq, struct libeth_xdpsq *sq __always_unused) +{ + ixgbevf_xdp_tx_put_nq(xdpsq); +} + +static inline void ixgbevf_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush) +{ + struct ixgbevf_ring *xdp_ring = xdpsq; + union ixgbe_adv_tx_desc *desc; + bool is_shared; + u32 ltu; + + is_shared = !test_bit(__IXGBEVF_TX_XDP_RING, &xdp_ring->state); + + if (is_shared) + ixgbevf_xdp_tx_get_nq(xdp_ring); + else + libeth_xdpsq_lock(&xdp_ring->xdpq_lock); + + if ((!flush && xdp_ring->pending < xdp_ring->count - 1) || + xdp_ring->cached_ntu == xdp_ring->next_to_use) + goto unlock; + + ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; + + /* We will not get DD on a context descriptor */ + if (unlikely(xdp_ring->xdp_sqes[ltu].type == LIBETH_SQE_CTX)) + goto unlock; + + desc = IXGBEVF_TX_DESC(xdp_ring, ltu); + desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); + + xdp_ring->xdp_sqes[xdp_ring->cached_ntu].rs_idx = ltu + 1; + xdp_ring->cached_ntu = xdp_ring->next_to_use; + + /* In case the packet was interrupted, discard it */ + xdp_ring->xdp_sqes[ltu].priv = 0; + + /* Finish descriptor writes before bumping tail */ + wmb(); + ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); + +unlock: + if (is_shared) + ixgbevf_xdp_tx_put_nq(xdp_ring); + else + libeth_xdpsq_unlock(&xdp_ring->xdpq_lock); +} + +#endif /* _IXGBEVF_XDP_LIB_H_ */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c new file mode 100644 index 00000000000000..f3edeae6aa00a6 --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2026 Intel Corporation */ + +#include + +#include "ixgbevf_txrx_lib.h" +#include "ixgbevf_xdp_lib.h" +#include "ixgbevf_xsk.h" + +/** + * ixgbevf_single_irq_disable - Mask off interrupt generation on a single vector + * @adapter: board private structure + * @vidx: vector id + **/ +static void ixgbevf_single_irq_disable(struct ixgbevf_adapter *adapter, + u16 vidx) +{ + struct ixgbe_hw *hw = &adapter->hw; + + IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, + adapter->eims_enable_mask & ~BIT(vidx)); + IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, BIT(vidx)); + IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, + adapter->eims_enable_mask & ~BIT(vidx)); + + IXGBE_WRITE_FLUSH(hw); + + synchronize_irq(adapter->msix_entries[vidx].vector); +} + +static void ixgbevf_qp_dis(struct ixgbevf_adapter *adapter, u16 qid) +{ + struct ixgbevf_ring *tx_ring, *rx_ring = adapter->rx_ring[qid]; + struct ixgbevf_q_vector *q_vector = rx_ring->q_vector; + + netif_stop_subqueue(adapter->netdev, qid); + ixgbevf_single_irq_disable(adapter, q_vector->v_idx); + napi_disable(&q_vector->napi); + + synchronize_net(); + + ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[qid]); + ixgbevf_clean_rx_ring(rx_ring); + ixgbevf_rx_destroy_pp(rx_ring); + + /* Clean both XDP and normal Tx queue */ + ixgbevf_for_each_ring(tx_ring, q_vector->tx) { + ixgbevf_flush_tx_queue(tx_ring); + if (ring_is_xdp(tx_ring)) + ixgbevf_clean_xdp_ring(tx_ring); + else + ixgbevf_clean_tx_ring(tx_ring); + } +} + +static void ixgbevf_qp_ena(struct ixgbevf_adapter *adapter, u16 qid) +{ + struct ixgbevf_ring *tx_ring, *rx_ring = adapter->rx_ring[qid]; + struct ixgbevf_q_vector *q_vector = rx_ring->q_vector; + + ixgbevf_configure_rx_ring(adapter, rx_ring); + ixgbevf_for_each_ring(tx_ring, q_vector->tx) + ixgbevf_configure_tx_ring(adapter, tx_ring); + + napi_enable(&q_vector->napi); + ixgbevf_irq_enable(adapter); + netif_start_subqueue(adapter->netdev, qid); + napi_schedule(&q_vector->napi); +} + +int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, + struct xsk_buff_pool *pool, u16 qid) +{ + bool running = !test_bit(__IXGBEVF_DOWN, &adapter->state) && + adapter->xdp_prog; + int err; + + if (running) + ixgbevf_qp_dis(adapter, qid); + + err = libeth_xsk_setup_pool(adapter->netdev, qid, !!pool); + + if (running) + ixgbevf_qp_ena(adapter, qid); + + return err; +} + +static void ixgbevf_fill_rx_xsk_desc(const struct libeth_xskfq_fp *fq, u32 i) +{ + union ixgbe_adv_rx_desc *rx_desc = + &((union ixgbe_adv_rx_desc *)fq->descs)[i]; + + rx_desc->read.pkt_addr = + cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i])); + rx_desc->wb.upper.length = 0; +} + +bool ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num) +{ + struct libeth_xskfq_fp fq = { + .count = rx_ring->count, + .descs = rx_ring->desc, + .fqes = rx_ring->xsk_fqes, + .ntu = rx_ring->next_to_use, + .pool = rx_ring->xsk_pool, + }; + u32 done; + + done = libeth_xskfqe_alloc(&fq, num, ixgbevf_fill_rx_xsk_desc); + if (likely(done)) { + /* Finish descriptor writes before bumping tail */ + wmb(); + ixgbevf_write_tail(rx_ring, fq.ntu); + } + + rx_ring->next_to_use = fq.ntu; + rx_ring->pending -= done; + + return done == num; +} + +void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring) +{ + u32 ntc = rx_ring->next_to_clean; + + if (rx_ring->xsk_xdp_head) + xsk_buff_free(&rx_ring->xsk_xdp_head->base); + + rx_ring->xsk_xdp_head = NULL; + + while (ntc != rx_ring->next_to_use) { + xsk_buff_free(&rx_ring->xsk_fqes[ntc]->base); + ntc++; + ntc = ntc == rx_ring->count ? 0 : ntc; + } +} + +struct ixgbevf_zc_sqe_priv { + u16 first_desc; + u16 len; +}; + +static_assert(sizeof(struct ixgbevf_zc_sqe_priv) <= + sizeof_field(struct libeth_sqe, priv)); + +static void ixgbevf_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, u64 priv) +{ + union ixgbe_adv_tx_desc *descs = sq->descs, *tx_desc = &descs[i]; + u32 ltu = (i ? : sq->count) - 1; + + u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS | + desc.len; + + tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); + + if (likely((desc.flags & LIBETH_XDP_TX_LAST) && !sq->sqes[ltu].priv)) { + tx_desc->read.olinfo_status = + cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + tx_desc->read.cmd_type_len = + cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP); + return; + } + + /* No previous packet */ + if (!sq->sqes[ltu].priv) { + struct ixgbevf_zc_sqe_priv *sqe_priv = + (void *)&sq->sqes[i].priv; + + sqe_priv->first_desc = i; + sqe_priv->len = desc.len; + + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + + return; + } + + if (sq->sqes[ltu].priv) { + struct ixgbevf_zc_sqe_priv *sqe_priv = + (void *)&sq->sqes[i].priv; + + sq->sqes[i].priv = sq->sqes[ltu].priv; + sq->sqes[ltu].priv = 0; + sqe_priv->len += desc.len; + + if (desc.flags & LIBETH_XDP_TX_LAST) { + union ixgbe_adv_tx_desc *first_desc = + &descs[sqe_priv->first_desc]; + + first_desc->read.olinfo_status = + cpu_to_le32((sqe_priv->len << + IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + tx_desc->read.cmd_type_len = + cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP); + cmd_type |= IXGBE_TXD_CMD_EOP; + sq->sqes[i].priv = 0; + } + + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + } +} + +LIBETH_XDP_DEFINE_START(); +LIBETH_XSK_DEFINE_FLUSH_TX(static ixgbevf_xsk_flush_tx, ixgbevf_prep_xdp_sq, + ixgbevf_xsk_xmit_desc); +__LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xsk_flush_tx_shared, + ixgbevf_prep_tx_sq, ixgbevf_xsk_xmit_desc, + libeth_xdp_tx_unprep, xsk); +LIBETH_XDP_DEFINE_END(); + +static bool ixgbevf_xsk_flush_common(struct libeth_xdp_tx_bulk *bq, u32 flags) +{ + const struct ixgbevf_ring *ring = bq->xdpsq; + + if (test_bit(__IXGBEVF_TX_XDP_RING, &ring->state)) + return ixgbevf_xsk_flush_tx(bq, flags); + else + return ixgbevf_xsk_flush_tx_shared(bq, flags); +} + +LIBETH_XDP_DEFINE_START(); +LIBETH_XSK_DEFINE_RUN_PROG(static ixgbevf_xsk_run_prog, + ixgbevf_xsk_flush_common); +LIBETH_XSK_DEFINE_FINALIZE(static ixgbevf_xsk_finalize_xdp_napi, + ixgbevf_xsk_flush_common, ixgbevf_xdp_rs_and_bump); +LIBETH_XDP_DEFINE_END(); + +u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *rx_ring, int budget) +{ + struct ixgbevf_adapter *adapter = q_vector->adapter; + int num_xdp_queues = adapter->num_xdp_queues; + u32 total_rx_bytes = 0, total_rx_packets = 0; + LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); + struct libeth_xdp_buff *head_xdp; + bool failure = false, wake; + struct sk_buff *skb; + + wake = xsk_uses_need_wakeup(rx_ring->xsk_pool); + if (wake) + xsk_clear_rx_need_wakeup(rx_ring->xsk_pool); + + if (num_xdp_queues) + libeth_xsk_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, + adapter->netdev, adapter->xdp_ring, + num_xdp_queues); + else + libeth_xsk_tx_init_bulk_shared(&xdp_tx_bulk, rx_ring->xdp_prog, + adapter->netdev, + adapter->tx_ring, + adapter->num_tx_queues); + + while (likely(total_rx_packets < budget)) { + union ixgbe_adv_rx_desc *rx_desc; + struct libeth_xdp_buff *rx_buffer; + unsigned int size; + u32 xdp_result; + + rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); + size = le16_to_cpu(rx_desc->wb.upper.length); + if (unlikely(!size)) + break; + + /* Avoid reading other descriptor fields before checking size */ + rmb(); + + rx_buffer = rx_ring->xsk_fqes[rx_ring->next_to_clean]; + head_xdp = libeth_xsk_process_buff(head_xdp, rx_buffer, size); + if (unlikely(!head_xdp) || ixgbevf_is_non_eop(rx_ring, rx_desc)) + continue; + + total_rx_packets++; + total_rx_bytes += xdp_get_buff_len(&head_xdp->base); + + xdp_result = ixgbevf_xsk_run_prog(head_xdp, &xdp_tx_bulk); + if (xdp_result) { + head_xdp = NULL; + if (likely(xdp_result != LIBETH_XDP_ABORTED)) + continue; + failure = true; + break; + } + + skb = xdp_build_skb_from_zc(&head_xdp->base); + + if (unlikely(!skb)) { + libeth_xdp_return_buff_slow(head_xdp); + head_xdp = NULL; + rx_ring->rx_stats.alloc_rx_buff_failed++; + break; + } + + head_xdp = NULL; + + if (unlikely(ixgbevf_cleanup_headers(rx_ring, rx_desc, skb))) { + skb = NULL; + continue; + } + + if (unlikely((skb->pkt_type == PACKET_BROADCAST || + skb->pkt_type == PACKET_MULTICAST) && + ether_addr_equal(rx_ring->netdev->dev_addr, + eth_hdr(skb)->h_source))) { + dev_kfree_skb_irq(skb); + continue; + } + + /* populate checksum, VLAN, and protocol */ + ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); + + napi_gro_receive(&q_vector->napi, skb); + } + + if (rx_ring->pending >= rx_ring->thresh) + failure |= !ixgbevf_xsk_alloc_rx_bufs(rx_ring, + rx_ring->pending); + + /* place incomplete frames back on ring for completion */ + rx_ring->xsk_xdp_head = head_xdp; + + ixgbevf_xsk_finalize_xdp_napi(&xdp_tx_bulk); + + u64_stats_update_begin(&rx_ring->syncp); + rx_ring->stats.packets += total_rx_packets; + rx_ring->stats.bytes += total_rx_bytes; + u64_stats_update_end(&rx_ring->syncp); + q_vector->rx.total_packets += total_rx_packets; + q_vector->rx.total_bytes += total_rx_bytes; + + if (likely(!failure)) + return total_rx_packets; + + if (wake) + xsk_set_rx_need_wakeup(rx_ring->xsk_pool); + + return budget; +} + +bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *tx_ring, int napi_budget) +{ + u32 budget = min_t(u32, napi_budget, tx_ring->thresh); + + if (ring_is_xdp(tx_ring)) + return libeth_xsk_xmit_do_bulk(tx_ring->xsk_pool, tx_ring, + budget, NULL, + ixgbevf_prep_xdp_sq, + ixgbevf_xsk_xmit_desc, + ixgbevf_xdp_rs_and_bump); + else + return __libeth_xsk_xmit_do_bulk(tx_ring->xsk_pool, tx_ring, + budget, NULL, + ixgbevf_prep_tx_sq, + ixgbevf_xsk_xmit_desc, + ixgbevf_xdp_rs_and_bump, + ixgbevf_xdp_tx_unprep); +} + +int ixgbevf_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) +{ + struct ixgbevf_adapter *adapter = netdev_priv(dev); + struct ixgbevf_q_vector *q_vector; + struct ixgbevf_ring *rx_ring; + + if (unlikely(test_bit(__IXGBEVF_DOWN, &adapter->state))) + return -ENETDOWN; + + rx_ring = adapter->rx_ring[queue_id]; + if (unlikely(!ring_is_xsk(rx_ring))) + return -EINVAL; + + q_vector = rx_ring->q_vector; + if (!napi_if_scheduled_mark_missed(&q_vector->napi)) + IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, + BIT(q_vector->v_idx)); + + return 0; +} diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h new file mode 100644 index 00000000000000..2bb39735b10efd --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2026 Intel Corporation */ + +#ifndef _IXGBEVF_XSK_H_ +#define _IXGBEVF_XSK_H_ + +/* Process completions as soon as possible */ +#define IXGBEVF_XSK_TX_CLEAN_THRESH(r) ((r)->count - 1) +#define IXGBEVF_XSK_MAX_ZC_FRAGS min(18, MAX_SKB_FRAGS) + +int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, + struct xsk_buff_pool *pool, u16 qid); +bool ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num); +void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring); +u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *rx_ring, int budget); +bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *tx_ring, int napi_budget); +int ixgbevf_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags); + +#endif /* _IXGBEVF_XSK_H_ */ diff --git a/include/net/libeth/tx.h b/include/net/libeth/tx.h index c3db5c6f16410c..f3edefe2411da4 100644 --- a/include/net/libeth/tx.h +++ b/include/net/libeth/tx.h @@ -74,7 +74,7 @@ struct libeth_sqe { u32 packets; u32 bytes; - unsigned long priv; + u64 priv; } __aligned_largest; /** diff --git a/include/net/libeth/xdp.h b/include/net/libeth/xdp.h index 898723ab62e812..92bc82f5dc05e9 100644 --- a/include/net/libeth/xdp.h +++ b/include/net/libeth/xdp.h @@ -132,17 +132,18 @@ static inline bool libeth_xdpsq_shared(u32 num) /** * libeth_xdpsq_id - get XDPSQ index corresponding to this CPU * @num: number of active XDPSQs + * @force_share: always share queues * * Helper for libeth_xdp routines, do not use in drivers directly. * * Return: XDPSQ index needs to be used on this CPU. */ -static inline u32 libeth_xdpsq_id(u32 num) +static inline u32 libeth_xdpsq_id(u32 num, bool force_share) { u32 ret = raw_smp_processor_id(); - if (static_branch_unlikely(&libeth_xdpsq_share) && - libeth_xdpsq_shared(num)) + if ((static_branch_unlikely(&libeth_xdpsq_share) && + libeth_xdpsq_shared(num)) || force_share) ret %= num; return ret; @@ -521,7 +522,8 @@ libeth_xdp_tx_xmit_bulk(const struct libeth_xdp_tx_frame *bulk, void *xdpsq, (*fill)(struct libeth_xdp_tx_frame frm, u32 i, const struct libeth_xdpsq *sq, u64 priv), void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i, - const struct libeth_xdpsq *sq, u64 priv)) + const struct libeth_xdpsq *sq, u64 priv), + void (*unprep)(void *xdpsq, struct libeth_xdpsq *sq)) { struct libeth_xdpsq sq __uninitialized; u32 this, batched, off = 0; @@ -579,7 +581,7 @@ libeth_xdp_tx_xmit_bulk(const struct libeth_xdp_tx_frame *bulk, void *xdpsq, *sq.xdp_tx += n; unlock: - libeth_xdpsq_unlock(sq.lock); + unprep(xdpsq, &sq); return n; } @@ -775,14 +777,15 @@ __libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags, const struct libeth_xdpsq *sq, u64 priv), void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i, const struct libeth_xdpsq *sq, - u64 priv)) + u64 priv), + void (*unprep)(void *xdpsq, struct libeth_xdpsq *sq)) { u32 sent, drops; int err = 0; sent = libeth_xdp_tx_xmit_bulk(bq->bulk, bq->xdpsq, min(bq->count, LIBETH_XDP_TX_BULK), - false, 0, prep, fill, xmit); + false, 0, prep, fill, xmit, unprep); drops = bq->count - sent; if (unlikely(drops)) { @@ -807,9 +810,9 @@ __libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags, * Use via LIBETH_XDP_DEFINE_FLUSH_TX() to define an ``XDP_TX`` driver * callback. */ -#define libeth_xdp_tx_flush_bulk(bq, flags, prep, xmit) \ +#define libeth_xdp_tx_flush_bulk(bq, flags, prep, xmit, unprep) \ __libeth_xdp_tx_flush_bulk(bq, flags, prep, libeth_xdp_tx_fill_buf, \ - xmit) + xmit, unprep) /* .ndo_xdp_xmit() implementation */ @@ -821,7 +824,12 @@ __libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags, * @num: number of active XDPSQs (the above array length) */ #define libeth_xdp_xmit_init_bulk(bq, dev, xdpsqs, num) \ - __libeth_xdp_xmit_init_bulk(bq, dev, (xdpsqs)[libeth_xdpsq_id(num)]) + __libeth_xdp_xmit_init_bulk(bq, dev, \ + (xdpsqs)[libeth_xdpsq_id(num, false)]) + +#define libeth_xdp_xmit_init_bulk_shared(bq, dev, xdpsqs, num) \ + __libeth_xdp_xmit_init_bulk(bq, dev, \ + (xdpsqs)[libeth_xdpsq_id(num, true)]) static inline void __libeth_xdp_xmit_init_bulk(struct libeth_xdp_tx_bulk *bq, struct net_device *dev, @@ -1017,6 +1025,12 @@ libeth_xdp_xmit_fill_buf(struct libeth_xdp_tx_frame frm, u32 i, return desc; } +static inline void libeth_xdp_tx_unprep(void *xdpsq __always_unused, + struct libeth_xdpsq *sq) +{ + libeth_xdpsq_unlock(sq->lock); +} + /** * libeth_xdp_xmit_flush_bulk - wrapper to define flush of one XDP xmit bulk * @bq: bulk to flush @@ -1027,9 +1041,10 @@ libeth_xdp_xmit_fill_buf(struct libeth_xdp_tx_frame frm, u32 i, * Use via LIBETH_XDP_DEFINE_FLUSH_XMIT() to define an XDP xmit driver * callback. */ -#define libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit) \ +#define libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit, unprep) \ __libeth_xdp_tx_flush_bulk(bq, (flags) | LIBETH_XDP_TX_NDO, prep, \ - libeth_xdp_xmit_fill_buf, xmit) + libeth_xdp_xmit_fill_buf, xmit, \ + unprep) u32 libeth_xdp_xmit_return_bulk(const struct libeth_xdp_tx_frame *bq, u32 count, const struct net_device *dev); @@ -1094,7 +1109,7 @@ __libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq, * @xqs: array of XDPSQs driver structs * @nqs: number of active XDPSQs, the above array length * @fl: driver callback to flush an XDP xmit bulk - * @fin: driver cabback to finalize the queue + * @fin: driver callback to finalize the queue * * If the driver has active XDPSQs, perform common checks and send the frames. * Finalize the queue, if requested. @@ -1139,10 +1154,15 @@ __libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq, * Do not use for XSk, it has its own optimized helper. */ #define libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num) \ - __libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false, \ + __libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false, false, \ __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_)) -#define __libeth_xdp_tx_init_bulk(bq, pr, d, xdpsqs, num, xsk, ub, un) do { \ +#define libeth_xdp_tx_init_bulk_shared(bq, prog, dev, xdpsqs, num) \ + __libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false, true, \ + __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_)) + +#define __libeth_xdp_tx_init_bulk(bq, pr, d, xdpsqs, num, xsk, fs, ub, un) \ +do { \ typeof(bq) ub = (bq); \ u32 un = (num); \ \ @@ -1151,7 +1171,7 @@ __libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq, if (un || (xsk)) { \ ub->prog = rcu_dereference(pr); \ ub->dev = (d); \ - ub->xdpsq = (xdpsqs)[libeth_xdpsq_id(un)]; \ + ub->xdpsq = (xdpsqs)[libeth_xdpsq_id(un, fs)]; \ } else { \ ub->prog = NULL; \ } \ @@ -1611,12 +1631,13 @@ void name(struct work_struct *work) \ * @xmit: driver callback to write a HW Tx descriptor */ #define LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit) \ - __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xdp) + __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, libeth_xdp_tx_unprep, \ + xdp) -#define __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, pfx) \ +#define __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, unprep, pfx) \ bool name(struct libeth_xdp_tx_bulk *bq, u32 flags) \ { \ - return libeth_##pfx##_tx_flush_bulk(bq, flags, prep, xmit); \ + return libeth_##pfx##_tx_flush_bulk(bq, flags, prep, xmit, unprep); \ } /** @@ -1626,9 +1647,12 @@ bool name(struct libeth_xdp_tx_bulk *bq, u32 flags) \ * @xmit: driver callback to write a HW Tx descriptor */ #define LIBETH_XDP_DEFINE_FLUSH_XMIT(name, prep, xmit) \ + __LIBETH_XDP_DEFINE_FLUSH_XMIT(name, prep, xmit, libeth_xdp_tx_unprep) + +#define __LIBETH_XDP_DEFINE_FLUSH_XMIT(name, prep, xmit, unprep) \ bool name(struct libeth_xdp_tx_bulk *bq, u32 flags) \ { \ - return libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit); \ + return libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit, unprep); \ } /** diff --git a/include/net/libeth/xsk.h b/include/net/libeth/xsk.h index 82b5d21aae8784..fa66b740994ca4 100644 --- a/include/net/libeth/xsk.h +++ b/include/net/libeth/xsk.h @@ -146,9 +146,10 @@ libeth_xsk_tx_fill_buf(struct libeth_xdp_tx_frame frm, u32 i, * Use via LIBETH_XSK_DEFINE_FLUSH_TX() to define an XSk ``XDP_TX`` driver * callback. */ -#define libeth_xsk_tx_flush_bulk(bq, flags, prep, xmit) \ +#define libeth_xsk_tx_flush_bulk(bq, flags, prep, xmit, unprep) \ __libeth_xdp_tx_flush_bulk(bq, (flags) | LIBETH_XDP_TX_XSK, prep, \ - libeth_xsk_tx_fill_buf, xmit) + libeth_xsk_tx_fill_buf, xmit, \ + unprep) /* XSk TMO */ @@ -282,12 +283,13 @@ libeth_xsk_xmit_fill_buf(struct libeth_xdp_tx_frame frm, u32 i, * Return: false if @budget was exhausted, true otherwise. */ static __always_inline bool -libeth_xsk_xmit_do_bulk(struct xsk_buff_pool *pool, void *xdpsq, u32 budget, - const struct xsk_tx_metadata_ops *tmo, - u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq), - void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i, - const struct libeth_xdpsq *sq, u64 priv), - void (*finalize)(void *xdpsq, bool sent, bool flush)) +__libeth_xsk_xmit_do_bulk(struct xsk_buff_pool *pool, void *xdpsq, u32 budget, + const struct xsk_tx_metadata_ops *tmo, + u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq), + void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, u64 priv), + void (*finalize)(void *xdpsq, bool sent, bool flush), + void (*unprep)(void *xdpsq, struct libeth_xdpsq *sq)) { const struct libeth_xdp_tx_frame *bulk; bool wake; @@ -298,11 +300,14 @@ libeth_xsk_xmit_do_bulk(struct xsk_buff_pool *pool, void *xdpsq, u32 budget, xsk_clear_tx_need_wakeup(pool); n = xsk_tx_peek_release_desc_batch(pool, budget); + if (unlikely(!n)) + return true; bulk = container_of(&pool->tx_descs[0], typeof(*bulk), desc); libeth_xdp_tx_xmit_bulk(bulk, xdpsq, n, true, libeth_xdp_ptr_to_priv(tmo), prep, - libeth_xsk_xmit_fill_buf, xmit); + libeth_xsk_xmit_fill_buf, xmit, + unprep); finalize(xdpsq, n, true); if (wake) @@ -311,6 +316,10 @@ libeth_xsk_xmit_do_bulk(struct xsk_buff_pool *pool, void *xdpsq, u32 budget, return n < budget; } +#define libeth_xsk_xmit_do_bulk(pool, xdpsq, budget, tmo, prep, xmit, finalize)\ + __libeth_xsk_xmit_do_bulk(pool, xdpsq, budget, tmo, prep, xmit, \ + finalize, libeth_xdp_tx_unprep) + /* Rx polling path */ /** @@ -327,7 +336,11 @@ libeth_xsk_xmit_do_bulk(struct xsk_buff_pool *pool, void *xdpsq, u32 budget, * when hitting this path. */ #define libeth_xsk_tx_init_bulk(bq, prog, dev, xdpsqs, num) \ - __libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, true, \ + __libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, true, false, \ + __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_)) + +#define libeth_xsk_tx_init_bulk_shared(bq, prog, dev, xdpsqs, num) \ + __libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, true, true, \ __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_)) struct libeth_xdp_buff *libeth_xsk_buff_add_frag(struct libeth_xdp_buff *head, @@ -546,7 +559,8 @@ __libeth_xsk_run_pass(struct libeth_xdp_buff *xdp, * @xmit: driver callback to write a HW Tx descriptor */ #define LIBETH_XSK_DEFINE_FLUSH_TX(name, prep, xmit) \ - __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xsk) + __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, libeth_xdp_tx_unprep, \ + xsk) /** * LIBETH_XSK_DEFINE_RUN_PROG - define a driver XDP program run function