diff options
Diffstat (limited to 'drivers/net/ethernet/cisco')
19 files changed, 810 insertions, 516 deletions
diff --git a/drivers/net/ethernet/cisco/enic/Kconfig b/drivers/net/ethernet/cisco/enic/Kconfig index ad80c0fa96a6..96709875fe4f 100644 --- a/drivers/net/ethernet/cisco/enic/Kconfig +++ b/drivers/net/ethernet/cisco/enic/Kconfig @@ -6,5 +6,6 @@ config ENIC tristate "Cisco VIC Ethernet NIC Support" depends on PCI + select PAGE_POOL help This enables the support for the Cisco VIC Ethernet card. diff --git a/drivers/net/ethernet/cisco/enic/Makefile b/drivers/net/ethernet/cisco/enic/Makefile index c3b6febfdbe4..a96b8332e6e2 100644 --- a/drivers/net/ethernet/cisco/enic/Makefile +++ b/drivers/net/ethernet/cisco/enic/Makefile @@ -3,5 +3,5 @@ obj-$(CONFIG_ENIC) := enic.o enic-y := enic_main.o vnic_cq.o vnic_intr.o vnic_wq.o \ enic_res.o enic_dev.o enic_pp.o vnic_dev.o vnic_rq.o vnic_vic.o \ - enic_ethtool.o enic_api.o enic_clsf.o + enic_ethtool.o enic_api.o enic_clsf.o enic_rq.o enic_wq.o diff --git a/drivers/net/ethernet/cisco/enic/cq_desc.h b/drivers/net/ethernet/cisco/enic/cq_desc.h index 462c5435a206..bfb3f14e89f5 100644 --- a/drivers/net/ethernet/cisco/enic/cq_desc.h +++ b/drivers/net/ethernet/cisco/enic/cq_desc.h @@ -40,28 +40,7 @@ struct cq_desc { #define CQ_DESC_COMP_NDX_BITS 12 #define CQ_DESC_COMP_NDX_MASK ((1 << CQ_DESC_COMP_NDX_BITS) - 1) -static inline void cq_desc_dec(const struct cq_desc *desc_arg, - u8 *type, u8 *color, u16 *q_number, u16 *completed_index) -{ - const struct cq_desc *desc = desc_arg; - const u8 type_color = desc->type_color; - - *color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK; - - /* - * Make sure color bit is read from desc *before* other fields - * are read from desc. Hardware guarantees color bit is last - * bit (byte) written. Adding the rmb() prevents the compiler - * and/or CPU from reordering the reads which would potentially - * result in reading stale values. - */ - - rmb(); - - *type = type_color & CQ_DESC_TYPE_MASK; - *q_number = le16_to_cpu(desc->q_number) & CQ_DESC_Q_NUM_MASK; - *completed_index = le16_to_cpu(desc->completed_index) & - CQ_DESC_COMP_NDX_MASK; -} +#define CQ_DESC_32_FI_MASK (BIT(0) | BIT(1)) +#define CQ_DESC_64_FI_MASK (BIT(0) | BIT(1)) #endif /* _CQ_DESC_H_ */ diff --git a/drivers/net/ethernet/cisco/enic/cq_enet_desc.h b/drivers/net/ethernet/cisco/enic/cq_enet_desc.h index d25426470a29..50787cff29db 100644 --- a/drivers/net/ethernet/cisco/enic/cq_enet_desc.h +++ b/drivers/net/ethernet/cisco/enic/cq_enet_desc.h @@ -17,12 +17,22 @@ struct cq_enet_wq_desc { u8 type_color; }; -static inline void cq_enet_wq_desc_dec(struct cq_enet_wq_desc *desc, - u8 *type, u8 *color, u16 *q_number, u16 *completed_index) -{ - cq_desc_dec((struct cq_desc *)desc, type, - color, q_number, completed_index); -} +/* + * Defines and Capabilities for CMD_CQ_ENTRY_SIZE_SET + */ +#define VNIC_RQ_ALL (~0ULL) + +#define VNIC_RQ_CQ_ENTRY_SIZE_16 0 +#define VNIC_RQ_CQ_ENTRY_SIZE_32 1 +#define VNIC_RQ_CQ_ENTRY_SIZE_64 2 + +#define VNIC_RQ_CQ_ENTRY_SIZE_16_CAPABLE BIT(VNIC_RQ_CQ_ENTRY_SIZE_16) +#define VNIC_RQ_CQ_ENTRY_SIZE_32_CAPABLE BIT(VNIC_RQ_CQ_ENTRY_SIZE_32) +#define VNIC_RQ_CQ_ENTRY_SIZE_64_CAPABLE BIT(VNIC_RQ_CQ_ENTRY_SIZE_64) + +#define VNIC_RQ_CQ_ENTRY_SIZE_ALL_BIT (VNIC_RQ_CQ_ENTRY_SIZE_16_CAPABLE | \ + VNIC_RQ_CQ_ENTRY_SIZE_32_CAPABLE | \ + VNIC_RQ_CQ_ENTRY_SIZE_64_CAPABLE) /* Completion queue descriptor: Ethernet receive queue, 16B */ struct cq_enet_rq_desc { @@ -36,6 +46,45 @@ struct cq_enet_rq_desc { u8 type_color; }; +/* Completion queue descriptor: Ethernet receive queue, 32B */ +struct cq_enet_rq_desc_32 { + __le16 completed_index_flags; + __le16 q_number_rss_type_flags; + __le32 rss_hash; + __le16 bytes_written_flags; + __le16 vlan; + __le16 checksum_fcoe; + u8 flags; + u8 fetch_index_flags; + __le32 time_stamp; + __le16 time_stamp2; + __le16 pie_info; + __le32 pie_info2; + __le16 pie_info3; + u8 pie_info4; + u8 type_color; +}; + +/* Completion queue descriptor: Ethernet receive queue, 64B */ +struct cq_enet_rq_desc_64 { + __le16 completed_index_flags; + __le16 q_number_rss_type_flags; + __le32 rss_hash; + __le16 bytes_written_flags; + __le16 vlan; + __le16 checksum_fcoe; + u8 flags; + u8 fetch_index_flags; + __le32 time_stamp; + __le16 time_stamp2; + __le16 pie_info; + __le32 pie_info2; + __le16 pie_info3; + u8 pie_info4; + u8 reserved[32]; + u8 type_color; +}; + #define CQ_ENET_RQ_DESC_FLAGS_INGRESS_PORT (0x1 << 12) #define CQ_ENET_RQ_DESC_FLAGS_FCOE (0x1 << 13) #define CQ_ENET_RQ_DESC_FLAGS_EOP (0x1 << 14) @@ -88,85 +137,4 @@ struct cq_enet_rq_desc { #define CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT (0x1 << 6) #define CQ_ENET_RQ_DESC_FLAGS_FCS_OK (0x1 << 7) -static inline void cq_enet_rq_desc_dec(struct cq_enet_rq_desc *desc, - u8 *type, u8 *color, u16 *q_number, u16 *completed_index, - u8 *ingress_port, u8 *fcoe, u8 *eop, u8 *sop, u8 *rss_type, - u8 *csum_not_calc, u32 *rss_hash, u16 *bytes_written, u8 *packet_error, - u8 *vlan_stripped, u16 *vlan_tci, u16 *checksum, u8 *fcoe_sof, - u8 *fcoe_fc_crc_ok, u8 *fcoe_enc_error, u8 *fcoe_eof, - u8 *tcp_udp_csum_ok, u8 *udp, u8 *tcp, u8 *ipv4_csum_ok, - u8 *ipv6, u8 *ipv4, u8 *ipv4_fragment, u8 *fcs_ok) -{ - u16 completed_index_flags; - u16 q_number_rss_type_flags; - u16 bytes_written_flags; - - cq_desc_dec((struct cq_desc *)desc, type, - color, q_number, completed_index); - - completed_index_flags = le16_to_cpu(desc->completed_index_flags); - q_number_rss_type_flags = - le16_to_cpu(desc->q_number_rss_type_flags); - bytes_written_flags = le16_to_cpu(desc->bytes_written_flags); - - *ingress_port = (completed_index_flags & - CQ_ENET_RQ_DESC_FLAGS_INGRESS_PORT) ? 1 : 0; - *fcoe = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_FCOE) ? - 1 : 0; - *eop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_EOP) ? - 1 : 0; - *sop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_SOP) ? - 1 : 0; - - *rss_type = (u8)((q_number_rss_type_flags >> CQ_DESC_Q_NUM_BITS) & - CQ_ENET_RQ_DESC_RSS_TYPE_MASK); - *csum_not_calc = (q_number_rss_type_flags & - CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ? 1 : 0; - - *rss_hash = le32_to_cpu(desc->rss_hash); - - *bytes_written = bytes_written_flags & - CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; - *packet_error = (bytes_written_flags & - CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ? 1 : 0; - *vlan_stripped = (bytes_written_flags & - CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) ? 1 : 0; - - /* - * Tag Control Information(16) = user_priority(3) + cfi(1) + vlan(12) - */ - *vlan_tci = le16_to_cpu(desc->vlan); - - if (*fcoe) { - *fcoe_sof = (u8)(le16_to_cpu(desc->checksum_fcoe) & - CQ_ENET_RQ_DESC_FCOE_SOF_MASK); - *fcoe_fc_crc_ok = (desc->flags & - CQ_ENET_RQ_DESC_FCOE_FC_CRC_OK) ? 1 : 0; - *fcoe_enc_error = (desc->flags & - CQ_ENET_RQ_DESC_FCOE_ENC_ERROR) ? 1 : 0; - *fcoe_eof = (u8)((le16_to_cpu(desc->checksum_fcoe) >> - CQ_ENET_RQ_DESC_FCOE_EOF_SHIFT) & - CQ_ENET_RQ_DESC_FCOE_EOF_MASK); - *checksum = 0; - } else { - *fcoe_sof = 0; - *fcoe_fc_crc_ok = 0; - *fcoe_enc_error = 0; - *fcoe_eof = 0; - *checksum = le16_to_cpu(desc->checksum_fcoe); - } - - *tcp_udp_csum_ok = - (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ? 1 : 0; - *udp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_UDP) ? 1 : 0; - *tcp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP) ? 1 : 0; - *ipv4_csum_ok = - (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ? 1 : 0; - *ipv6 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV6) ? 1 : 0; - *ipv4 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4) ? 1 : 0; - *ipv4_fragment = - (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT) ? 1 : 0; - *fcs_ok = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) ? 1 : 0; -} - #endif /* _CQ_ENET_DESC_H_ */ diff --git a/drivers/net/ethernet/cisco/enic/enic.h b/drivers/net/ethernet/cisco/enic/enic.h index 10b7e02ba4d0..9c12e967e9f1 100644 --- a/drivers/net/ethernet/cisco/enic/enic.h +++ b/drivers/net/ethernet/cisco/enic/enic.h @@ -17,6 +17,7 @@ #include "vnic_nic.h" #include "vnic_rss.h" #include <linux/irq.h> +#include <net/page_pool/helpers.h> #define DRV_NAME "enic" #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Driver" @@ -30,6 +31,13 @@ #define ENIC_AIC_LARGE_PKT_DIFF 3 +enum ext_cq { + ENIC_RQ_CQ_ENTRY_SIZE_16, + ENIC_RQ_CQ_ENTRY_SIZE_32, + ENIC_RQ_CQ_ENTRY_SIZE_64, + ENIC_RQ_CQ_ENTRY_SIZE_MAX, +}; + struct enic_msix_entry { int requested; char devname[IFNAMSIZ + 8]; @@ -75,6 +83,10 @@ struct enic_rx_coal { #define ENIC_SET_INSTANCE (1 << 3) #define ENIC_SET_HOST (1 << 4) +#define MAX_TSO BIT(16) +#define WQ_ENET_MAX_DESC_LEN BIT(WQ_ENET_LEN_BITS) +#define ENIC_DESC_MAX_SPLITS (MAX_TSO / WQ_ENET_MAX_DESC_LEN + 1) + struct enic_port_profile { u32 set; u8 request; @@ -158,6 +170,7 @@ struct enic_rq_stats { u64 pkt_truncated; /* truncated pkts */ u64 no_skb; /* out of skbs */ u64 desc_skip; /* Rx pkt went into later buffer */ + u64 pp_alloc_fail; /* page pool alloc failure */ }; struct enic_wq { @@ -169,6 +182,7 @@ struct enic_wq { struct enic_rq { struct vnic_rq vrq; struct enic_rq_stats stats; + struct page_pool *pool; } ____cacheline_aligned; /* Per-instance private data structure */ @@ -223,9 +237,9 @@ struct enic { unsigned int cq_avail; unsigned int cq_count; struct enic_rfs_flw_tbl rfs_h; - u32 rx_copybreak; u8 rss_key[ENIC_RSS_LEN]; struct vnic_gen_stats gen_stats; + enum ext_cq ext_cq; }; static inline struct net_device *vnic_get_netdev(struct vnic_dev *vdev) @@ -347,5 +361,6 @@ int enic_is_valid_vf(struct enic *enic, int vf); int enic_is_dynamic(struct enic *enic); void enic_set_ethtool_ops(struct net_device *netdev); int __enic_set_rsskey(struct enic *enic); +void enic_ext_cq(struct enic *enic); #endif /* _ENIC_H_ */ diff --git a/drivers/net/ethernet/cisco/enic/enic_clsf.h b/drivers/net/ethernet/cisco/enic/enic_clsf.h index 8c4ce50da6e1..5f5284102fb0 100644 --- a/drivers/net/ethernet/cisco/enic/enic_clsf.h +++ b/drivers/net/ethernet/cisco/enic/enic_clsf.h @@ -26,7 +26,7 @@ static inline void enic_rfs_timer_start(struct enic *enic) static inline void enic_rfs_timer_stop(struct enic *enic) { - del_timer_sync(&enic->rfs_h.rfs_may_expire); + timer_delete_sync(&enic->rfs_h.rfs_may_expire); } #else static inline void enic_rfs_timer_start(struct enic *enic) {} diff --git a/drivers/net/ethernet/cisco/enic/enic_ethtool.c b/drivers/net/ethernet/cisco/enic/enic_ethtool.c index d607b4f0542c..529160926a96 100644 --- a/drivers/net/ethernet/cisco/enic/enic_ethtool.c +++ b/drivers/net/ethernet/cisco/enic/enic_ethtool.c @@ -222,9 +222,9 @@ static void enic_get_ringparam(struct net_device *netdev, struct enic *enic = netdev_priv(netdev); struct vnic_enet_config *c = &enic->config; - ring->rx_max_pending = ENIC_MAX_RQ_DESCS; + ring->rx_max_pending = c->max_rq_ring; ring->rx_pending = c->rq_desc_count; - ring->tx_max_pending = ENIC_MAX_WQ_DESCS; + ring->tx_max_pending = c->max_wq_ring; ring->tx_pending = c->wq_desc_count; } @@ -252,18 +252,18 @@ static int enic_set_ringparam(struct net_device *netdev, } rx_pending = c->rq_desc_count; tx_pending = c->wq_desc_count; - if (ring->rx_pending > ENIC_MAX_RQ_DESCS || + if (ring->rx_pending > c->max_rq_ring || ring->rx_pending < ENIC_MIN_RQ_DESCS) { netdev_info(netdev, "rx pending (%u) not in range [%u,%u]", ring->rx_pending, ENIC_MIN_RQ_DESCS, - ENIC_MAX_RQ_DESCS); + c->max_rq_ring); return -EINVAL; } - if (ring->tx_pending > ENIC_MAX_WQ_DESCS || + if (ring->tx_pending > c->max_wq_ring || ring->tx_pending < ENIC_MIN_WQ_DESCS) { netdev_info(netdev, "tx pending (%u) not in range [%u,%u]", ring->tx_pending, ENIC_MIN_WQ_DESCS, - ENIC_MAX_WQ_DESCS); + c->max_wq_ring); return -EINVAL; } if (running) @@ -608,43 +608,6 @@ static int enic_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, return ret; } -static int enic_get_tunable(struct net_device *dev, - const struct ethtool_tunable *tuna, void *data) -{ - struct enic *enic = netdev_priv(dev); - int ret = 0; - - switch (tuna->id) { - case ETHTOOL_RX_COPYBREAK: - *(u32 *)data = enic->rx_copybreak; - break; - default: - ret = -EINVAL; - break; - } - - return ret; -} - -static int enic_set_tunable(struct net_device *dev, - const struct ethtool_tunable *tuna, - const void *data) -{ - struct enic *enic = netdev_priv(dev); - int ret = 0; - - switch (tuna->id) { - case ETHTOOL_RX_COPYBREAK: - enic->rx_copybreak = *(u32 *)data; - break; - default: - ret = -EINVAL; - break; - } - - return ret; -} - static u32 enic_get_rxfh_key_size(struct net_device *netdev) { return ENIC_RSS_LEN; @@ -727,8 +690,6 @@ static const struct ethtool_ops enic_ethtool_ops = { .get_coalesce = enic_get_coalesce, .set_coalesce = enic_set_coalesce, .get_rxnfc = enic_get_rxnfc, - .get_tunable = enic_get_tunable, - .set_tunable = enic_set_tunable, .get_rxfh_key_size = enic_get_rxfh_key_size, .get_rxfh = enic_get_rxfh, .set_rxfh = enic_set_rxfh, diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c index 49f6cab01ed5..c753c35b26eb 100644 --- a/drivers/net/ethernet/cisco/enic/enic_main.c +++ b/drivers/net/ethernet/cisco/enic/enic_main.c @@ -58,18 +58,15 @@ #include "enic_dev.h" #include "enic_pp.h" #include "enic_clsf.h" +#include "enic_rq.h" +#include "enic_wq.h" #define ENIC_NOTIFY_TIMER_PERIOD (2 * HZ) -#define WQ_ENET_MAX_DESC_LEN (1 << WQ_ENET_LEN_BITS) -#define MAX_TSO (1 << 16) -#define ENIC_DESC_MAX_SPLITS (MAX_TSO / WQ_ENET_MAX_DESC_LEN + 1) #define PCI_DEVICE_ID_CISCO_VIC_ENET 0x0043 /* ethernet vnic */ #define PCI_DEVICE_ID_CISCO_VIC_ENET_DYN 0x0044 /* enet dynamic vnic */ #define PCI_DEVICE_ID_CISCO_VIC_ENET_VF 0x0071 /* enet SRIOV VF */ -#define RX_COPYBREAK_DEFAULT 256 - /* Supported devices */ static const struct pci_device_id enic_id_table[] = { { PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET) }, @@ -322,54 +319,6 @@ int enic_is_valid_vf(struct enic *enic, int vf) #endif } -static void enic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf) -{ - struct enic *enic = vnic_dev_priv(wq->vdev); - - if (buf->sop) - dma_unmap_single(&enic->pdev->dev, buf->dma_addr, buf->len, - DMA_TO_DEVICE); - else - dma_unmap_page(&enic->pdev->dev, buf->dma_addr, buf->len, - DMA_TO_DEVICE); - - if (buf->os_buf) - dev_kfree_skb_any(buf->os_buf); -} - -static void enic_wq_free_buf(struct vnic_wq *wq, - struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque) -{ - struct enic *enic = vnic_dev_priv(wq->vdev); - - enic->wq[wq->index].stats.cq_work++; - enic->wq[wq->index].stats.cq_bytes += buf->len; - enic_free_wq_buf(wq, buf); -} - -static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, - u8 type, u16 q_number, u16 completed_index, void *opaque) -{ - struct enic *enic = vnic_dev_priv(vdev); - - spin_lock(&enic->wq[q_number].lock); - - vnic_wq_service(&enic->wq[q_number].vwq, cq_desc, - completed_index, enic_wq_free_buf, - opaque); - - if (netif_tx_queue_stopped(netdev_get_tx_queue(enic->netdev, q_number)) && - vnic_wq_desc_avail(&enic->wq[q_number].vwq) >= - (MAX_SKB_FRAGS + ENIC_DESC_MAX_SPLITS)) { - netif_wake_subqueue(enic->netdev, q_number); - enic->wq[q_number].stats.wake++; - } - - spin_unlock(&enic->wq[q_number].lock); - - return 0; -} - static bool enic_log_q_error(struct enic *enic) { unsigned int i; @@ -1313,243 +1262,6 @@ nla_put_failure: return -EMSGSIZE; } -static void enic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf) -{ - struct enic *enic = vnic_dev_priv(rq->vdev); - - if (!buf->os_buf) - return; - - dma_unmap_single(&enic->pdev->dev, buf->dma_addr, buf->len, - DMA_FROM_DEVICE); - dev_kfree_skb_any(buf->os_buf); - buf->os_buf = NULL; -} - -static int enic_rq_alloc_buf(struct vnic_rq *rq) -{ - struct enic *enic = vnic_dev_priv(rq->vdev); - struct net_device *netdev = enic->netdev; - struct sk_buff *skb; - unsigned int len = netdev->mtu + VLAN_ETH_HLEN; - unsigned int os_buf_index = 0; - dma_addr_t dma_addr; - struct vnic_rq_buf *buf = rq->to_use; - - if (buf->os_buf) { - enic_queue_rq_desc(rq, buf->os_buf, os_buf_index, buf->dma_addr, - buf->len); - - return 0; - } - skb = netdev_alloc_skb_ip_align(netdev, len); - if (!skb) { - enic->rq[rq->index].stats.no_skb++; - return -ENOMEM; - } - - dma_addr = dma_map_single(&enic->pdev->dev, skb->data, len, - DMA_FROM_DEVICE); - if (unlikely(enic_dma_map_check(enic, dma_addr))) { - dev_kfree_skb(skb); - return -ENOMEM; - } - - enic_queue_rq_desc(rq, skb, os_buf_index, - dma_addr, len); - - return 0; -} - -static void enic_intr_update_pkt_size(struct vnic_rx_bytes_counter *pkt_size, - u32 pkt_len) -{ - if (ENIC_LARGE_PKT_THRESHOLD <= pkt_len) - pkt_size->large_pkt_bytes_cnt += pkt_len; - else - pkt_size->small_pkt_bytes_cnt += pkt_len; -} - -static bool enic_rxcopybreak(struct net_device *netdev, struct sk_buff **skb, - struct vnic_rq_buf *buf, u16 len) -{ - struct enic *enic = netdev_priv(netdev); - struct sk_buff *new_skb; - - if (len > enic->rx_copybreak) - return false; - new_skb = netdev_alloc_skb_ip_align(netdev, len); - if (!new_skb) - return false; - dma_sync_single_for_cpu(&enic->pdev->dev, buf->dma_addr, len, - DMA_FROM_DEVICE); - memcpy(new_skb->data, (*skb)->data, len); - *skb = new_skb; - - return true; -} - -static void enic_rq_indicate_buf(struct vnic_rq *rq, - struct cq_desc *cq_desc, struct vnic_rq_buf *buf, - int skipped, void *opaque) -{ - struct enic *enic = vnic_dev_priv(rq->vdev); - struct net_device *netdev = enic->netdev; - struct sk_buff *skb; - struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)]; - struct enic_rq_stats *rqstats = &enic->rq[rq->index].stats; - - u8 type, color, eop, sop, ingress_port, vlan_stripped; - u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof; - u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok; - u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc; - u8 packet_error; - u16 q_number, completed_index, bytes_written, vlan_tci, checksum; - u32 rss_hash; - bool outer_csum_ok = true, encap = false; - - rqstats->packets++; - if (skipped) { - rqstats->desc_skip++; - return; - } - - skb = buf->os_buf; - - cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, - &type, &color, &q_number, &completed_index, - &ingress_port, &fcoe, &eop, &sop, &rss_type, - &csum_not_calc, &rss_hash, &bytes_written, - &packet_error, &vlan_stripped, &vlan_tci, &checksum, - &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error, - &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp, - &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment, - &fcs_ok); - - if (packet_error) { - - if (!fcs_ok) { - if (bytes_written > 0) - rqstats->bad_fcs++; - else if (bytes_written == 0) - rqstats->pkt_truncated++; - } - - dma_unmap_single(&enic->pdev->dev, buf->dma_addr, buf->len, - DMA_FROM_DEVICE); - dev_kfree_skb_any(skb); - buf->os_buf = NULL; - - return; - } - - if (eop && bytes_written > 0) { - - /* Good receive - */ - rqstats->bytes += bytes_written; - if (!enic_rxcopybreak(netdev, &skb, buf, bytes_written)) { - buf->os_buf = NULL; - dma_unmap_single(&enic->pdev->dev, buf->dma_addr, - buf->len, DMA_FROM_DEVICE); - } - prefetch(skb->data - NET_IP_ALIGN); - - skb_put(skb, bytes_written); - skb->protocol = eth_type_trans(skb, netdev); - skb_record_rx_queue(skb, q_number); - if ((netdev->features & NETIF_F_RXHASH) && rss_hash && - (type == 3)) { - switch (rss_type) { - case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv4: - case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6: - case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6_EX: - skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L4); - rqstats->l4_rss_hash++; - break; - case CQ_ENET_RQ_DESC_RSS_TYPE_IPv4: - case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6: - case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6_EX: - skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L3); - rqstats->l3_rss_hash++; - break; - } - } - if (enic->vxlan.vxlan_udp_port_number) { - switch (enic->vxlan.patch_level) { - case 0: - if (fcoe) { - encap = true; - outer_csum_ok = fcoe_fc_crc_ok; - } - break; - case 2: - if ((type == 7) && - (rss_hash & BIT(0))) { - encap = true; - outer_csum_ok = (rss_hash & BIT(1)) && - (rss_hash & BIT(2)); - } - break; - } - } - - /* Hardware does not provide whole packet checksum. It only - * provides pseudo checksum. Since hw validates the packet - * checksum but not provide us the checksum value. use - * CHECSUM_UNNECESSARY. - * - * In case of encap pkt tcp_udp_csum_ok/tcp_udp_csum_ok is - * inner csum_ok. outer_csum_ok is set by hw when outer udp - * csum is correct or is zero. - */ - if ((netdev->features & NETIF_F_RXCSUM) && !csum_not_calc && - tcp_udp_csum_ok && outer_csum_ok && - (ipv4_csum_ok || ipv6)) { - skb->ip_summed = CHECKSUM_UNNECESSARY; - skb->csum_level = encap; - if (encap) - rqstats->csum_unnecessary_encap++; - else - rqstats->csum_unnecessary++; - } - - if (vlan_stripped) { - __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci); - rqstats->vlan_stripped++; - } - skb_mark_napi_id(skb, &enic->napi[rq->index]); - if (!(netdev->features & NETIF_F_GRO)) - netif_receive_skb(skb); - else - napi_gro_receive(&enic->napi[q_number], skb); - if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce) - enic_intr_update_pkt_size(&cq->pkt_size_counter, - bytes_written); - } else { - - /* Buffer overflow - */ - rqstats->pkt_truncated++; - dma_unmap_single(&enic->pdev->dev, buf->dma_addr, buf->len, - DMA_FROM_DEVICE); - dev_kfree_skb_any(skb); - buf->os_buf = NULL; - } -} - -static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, - u8 type, u16 q_number, u16 completed_index, void *opaque) -{ - struct enic *enic = vnic_dev_priv(vdev); - - vnic_rq_service(&enic->rq[q_number].vrq, cq_desc, - completed_index, VNIC_RQ_RETURN_DESC, - enic_rq_indicate_buf, opaque); - - return 0; -} - static void enic_set_int_moderation(struct enic *enic, struct vnic_rq *rq) { unsigned int intr = enic_msix_rq_intr(enic, rq->index); @@ -1620,12 +1332,10 @@ static int enic_poll(struct napi_struct *napi, int budget) unsigned int work_done, rq_work_done = 0, wq_work_done; int err; - wq_work_done = vnic_cq_service(&enic->cq[cq_wq], wq_work_to_do, - enic_wq_service, NULL); + wq_work_done = enic_wq_cq_service(enic, cq_wq, wq_work_to_do); if (budget > 0) - rq_work_done = vnic_cq_service(&enic->cq[cq_rq], - rq_work_to_do, enic_rq_service, NULL); + rq_work_done = enic_rq_cq_service(enic, cq_rq, rq_work_to_do); /* Accumulate intr event credits for this polling * cycle. An intr event is the completion of a @@ -1724,8 +1434,8 @@ static int enic_poll_msix_wq(struct napi_struct *napi, int budget) wq_irq = wq->index; cq = enic_cq_wq(enic, wq_irq); intr = enic_msix_wq_intr(enic, wq_irq); - wq_work_done = vnic_cq_service(&enic->cq[cq], wq_work_to_do, - enic_wq_service, NULL); + + wq_work_done = enic_wq_cq_service(enic, cq, wq_work_to_do); vnic_intr_return_credits(&enic->intr[intr], wq_work_done, 0 /* don't unmask intr */, @@ -1754,8 +1464,7 @@ static int enic_poll_msix_rq(struct napi_struct *napi, int budget) */ if (budget > 0) - work_done = vnic_cq_service(&enic->cq[cq], - work_to_do, enic_rq_service, NULL); + work_done = enic_rq_cq_service(enic, cq, work_to_do); /* Return intr event credits for this polling * cycle. An intr event is the completion of a @@ -1972,6 +1681,17 @@ static int enic_open(struct net_device *netdev) struct enic *enic = netdev_priv(netdev); unsigned int i; int err, ret; + unsigned int max_pkt_len = netdev->mtu + VLAN_ETH_HLEN; + struct page_pool_params pp_params = { + .order = get_order(max_pkt_len), + .pool_size = enic->config.rq_desc_count, + .nid = dev_to_node(&enic->pdev->dev), + .dev = &enic->pdev->dev, + .dma_dir = DMA_FROM_DEVICE, + .max_len = (max_pkt_len > PAGE_SIZE) ? max_pkt_len : PAGE_SIZE, + .netdev = netdev, + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, + }; err = enic_request_intr(enic); if (err) { @@ -1989,6 +1709,16 @@ static int enic_open(struct net_device *netdev) } for (i = 0; i < enic->rq_count; i++) { + /* create a page pool for each RQ */ + pp_params.napi = &enic->napi[i]; + pp_params.queue_idx = i; + enic->rq[i].pool = page_pool_create(&pp_params); + if (IS_ERR(enic->rq[i].pool)) { + err = PTR_ERR(enic->rq[i].pool); + enic->rq[i].pool = NULL; + goto err_out_free_rq; + } + /* enable rq before updating rq desc */ vnic_rq_enable(&enic->rq[i].vrq); vnic_rq_fill(&enic->rq[i].vrq, enic_rq_alloc_buf); @@ -2029,8 +1759,11 @@ static int enic_open(struct net_device *netdev) err_out_free_rq: for (i = 0; i < enic->rq_count; i++) { ret = vnic_rq_disable(&enic->rq[i].vrq); - if (!ret) + if (!ret) { vnic_rq_clean(&enic->rq[i].vrq, enic_free_rq_buf); + page_pool_destroy(enic->rq[i].pool); + enic->rq[i].pool = NULL; + } } enic_dev_notify_unset(enic); err_out_free_intr: @@ -2054,7 +1787,7 @@ static int enic_stop(struct net_device *netdev) enic_synchronize_irqs(enic); - del_timer_sync(&enic->notify_timer); + timer_delete_sync(&enic->notify_timer); enic_rfs_flw_tbl_free(enic); enic_dev_disable(enic); @@ -2088,8 +1821,11 @@ static int enic_stop(struct net_device *netdev) for (i = 0; i < enic->wq_count; i++) vnic_wq_clean(&enic->wq[i].vwq, enic_free_wq_buf); - for (i = 0; i < enic->rq_count; i++) + for (i = 0; i < enic->rq_count; i++) { vnic_rq_clean(&enic->rq[i].vrq, enic_free_rq_buf); + page_pool_destroy(enic->rq[i].pool); + enic->rq[i].pool = NULL; + } for (i = 0; i < enic->cq_count; i++) vnic_cq_clean(&enic->cq[i]); for (i = 0; i < enic->intr_count; i++) @@ -2405,6 +2141,7 @@ static void enic_reset(struct work_struct *work) enic_init_vnic_resources(enic); enic_set_rss_nic_cfg(enic); enic_dev_set_ig_vlan_rewrite_mode(enic); + enic_ext_cq(enic); enic_open(enic->netdev); /* Allow infiniband to fiddle with the device again */ @@ -2431,6 +2168,7 @@ static void enic_tx_hang_reset(struct work_struct *work) enic_init_vnic_resources(enic); enic_set_rss_nic_cfg(enic); enic_dev_set_ig_vlan_rewrite_mode(enic); + enic_ext_cq(enic); enic_open(enic->netdev); /* Allow infiniband to fiddle with the device again */ @@ -2599,6 +2337,7 @@ static void enic_get_queue_stats_rx(struct net_device *dev, int idx, rxs->hw_drop_overruns = rqstats->pkt_truncated; rxs->csum_unnecessary = rqstats->csum_unnecessary + rqstats->csum_unnecessary_encap; + rxs->alloc_fail = rqstats->pp_alloc_fail; } static void enic_get_queue_stats_tx(struct net_device *dev, int idx, @@ -2626,6 +2365,7 @@ static void enic_get_base_stats(struct net_device *dev, rxs->hw_drops = 0; rxs->hw_drop_overruns = 0; rxs->csum_unnecessary = 0; + rxs->alloc_fail = 0; txs->bytes = 0; txs->packets = 0; txs->csum_none = 0; @@ -2803,6 +2543,8 @@ static int enic_dev_init(struct enic *enic) enic_get_res_counts(enic); + enic_ext_cq(enic); + err = enic_alloc_enic_resources(enic); if (err) { dev_err(dev, "Failed to allocate enic resources\n"); @@ -3179,7 +2921,6 @@ static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) dev_err(dev, "Cannot register net device, aborting\n"); goto err_out_dev_deinit; } - enic->rx_copybreak = RX_COPYBREAK_DEFAULT; return 0; diff --git a/drivers/net/ethernet/cisco/enic/enic_res.c b/drivers/net/ethernet/cisco/enic/enic_res.c index 126125199833..bbd3143ed73e 100644 --- a/drivers/net/ethernet/cisco/enic/enic_res.c +++ b/drivers/net/ethernet/cisco/enic/enic_res.c @@ -59,31 +59,38 @@ int enic_get_vnic_config(struct enic *enic) GET_CONFIG(intr_timer_usec); GET_CONFIG(loop_tag); GET_CONFIG(num_arfs); + GET_CONFIG(max_rq_ring); + GET_CONFIG(max_wq_ring); + GET_CONFIG(max_cq_ring); + + if (!c->max_wq_ring) + c->max_wq_ring = ENIC_MAX_WQ_DESCS_DEFAULT; + if (!c->max_rq_ring) + c->max_rq_ring = ENIC_MAX_RQ_DESCS_DEFAULT; + if (!c->max_cq_ring) + c->max_cq_ring = ENIC_MAX_CQ_DESCS_DEFAULT; c->wq_desc_count = - min_t(u32, ENIC_MAX_WQ_DESCS, - max_t(u32, ENIC_MIN_WQ_DESCS, - c->wq_desc_count)); + min_t(u32, c->max_wq_ring, + max_t(u32, ENIC_MIN_WQ_DESCS, c->wq_desc_count)); c->wq_desc_count &= 0xffffffe0; /* must be aligned to groups of 32 */ c->rq_desc_count = - min_t(u32, ENIC_MAX_RQ_DESCS, - max_t(u32, ENIC_MIN_RQ_DESCS, - c->rq_desc_count)); + min_t(u32, c->max_rq_ring, + max_t(u32, ENIC_MIN_RQ_DESCS, c->rq_desc_count)); c->rq_desc_count &= 0xffffffe0; /* must be aligned to groups of 32 */ if (c->mtu == 0) c->mtu = 1500; - c->mtu = min_t(u16, ENIC_MAX_MTU, - max_t(u16, ENIC_MIN_MTU, - c->mtu)); + c->mtu = min_t(u16, ENIC_MAX_MTU, max_t(u16, ENIC_MIN_MTU, c->mtu)); c->intr_timer_usec = min_t(u32, c->intr_timer_usec, vnic_dev_get_intr_coal_timer_max(enic->vdev)); dev_info(enic_get_dev(enic), - "vNIC MAC addr %pM wq/rq %d/%d mtu %d\n", - enic->mac_addr, c->wq_desc_count, c->rq_desc_count, c->mtu); + "vNIC MAC addr %pM wq/rq %d/%d max wq/rq/cq %d/%d/%d mtu %d\n", + enic->mac_addr, c->wq_desc_count, c->rq_desc_count, + c->max_wq_ring, c->max_rq_ring, c->max_cq_ring, c->mtu); dev_info(enic_get_dev(enic), "vNIC csum tx/rx %s/%s " "tso/lro %s/%s rss %s intr mode %s type %s timer %d usec " @@ -312,6 +319,7 @@ void enic_init_vnic_resources(struct enic *enic) int enic_alloc_vnic_resources(struct enic *enic) { enum vnic_dev_intr_mode intr_mode; + int rq_cq_desc_size; unsigned int i; int err; @@ -326,6 +334,24 @@ int enic_alloc_vnic_resources(struct enic *enic) intr_mode == VNIC_DEV_INTR_MODE_MSIX ? "MSI-X" : "unknown"); + switch (enic->ext_cq) { + case ENIC_RQ_CQ_ENTRY_SIZE_16: + rq_cq_desc_size = 16; + break; + case ENIC_RQ_CQ_ENTRY_SIZE_32: + rq_cq_desc_size = 32; + break; + case ENIC_RQ_CQ_ENTRY_SIZE_64: + rq_cq_desc_size = 64; + break; + default: + dev_err(enic_get_dev(enic), + "Unable to determine rq cq desc size: %d", + enic->ext_cq); + err = -ENODEV; + goto err_out; + } + /* Allocate queue resources */ @@ -348,8 +374,8 @@ int enic_alloc_vnic_resources(struct enic *enic) for (i = 0; i < enic->cq_count; i++) { if (i < enic->rq_count) err = vnic_cq_alloc(enic->vdev, &enic->cq[i], i, - enic->config.rq_desc_count, - sizeof(struct cq_enet_rq_desc)); + enic->config.rq_desc_count, + rq_cq_desc_size); else err = vnic_cq_alloc(enic->vdev, &enic->cq[i], i, enic->config.wq_desc_count, @@ -380,6 +406,39 @@ int enic_alloc_vnic_resources(struct enic *enic) err_out_cleanup: enic_free_vnic_resources(enic); - +err_out: return err; } + +/* + * CMD_CQ_ENTRY_SIZE_SET can fail on older hw generations that don't support + * that command + */ +void enic_ext_cq(struct enic *enic) +{ + u64 a0 = CMD_CQ_ENTRY_SIZE_SET, a1 = 0; + int wait = 1000; + int ret; + + spin_lock_bh(&enic->devcmd_lock); + ret = vnic_dev_cmd(enic->vdev, CMD_CAPABILITY, &a0, &a1, wait); + if (ret || a0) { + dev_info(&enic->pdev->dev, + "CMD_CQ_ENTRY_SIZE_SET not supported."); + enic->ext_cq = ENIC_RQ_CQ_ENTRY_SIZE_16; + goto out; + } + a1 &= VNIC_RQ_CQ_ENTRY_SIZE_ALL_BIT; + enic->ext_cq = fls(a1) - 1; + a0 = VNIC_RQ_ALL; + a1 = enic->ext_cq; + ret = vnic_dev_cmd(enic->vdev, CMD_CQ_ENTRY_SIZE_SET, &a0, &a1, wait); + if (ret) { + dev_info(&enic->pdev->dev, "CMD_CQ_ENTRY_SIZE_SET failed."); + enic->ext_cq = ENIC_RQ_CQ_ENTRY_SIZE_16; + } +out: + spin_unlock_bh(&enic->devcmd_lock); + dev_info(&enic->pdev->dev, "CQ entry size set to %d bytes", + 16 << enic->ext_cq); +} diff --git a/drivers/net/ethernet/cisco/enic/enic_res.h b/drivers/net/ethernet/cisco/enic/enic_res.h index b8ee42d297aa..02dca1ae4a22 100644 --- a/drivers/net/ethernet/cisco/enic/enic_res.h +++ b/drivers/net/ethernet/cisco/enic/enic_res.h @@ -12,10 +12,13 @@ #include "vnic_wq.h" #include "vnic_rq.h" -#define ENIC_MIN_WQ_DESCS 64 -#define ENIC_MAX_WQ_DESCS 4096 -#define ENIC_MIN_RQ_DESCS 64 -#define ENIC_MAX_RQ_DESCS 4096 +#define ENIC_MIN_WQ_DESCS 64 +#define ENIC_MAX_WQ_DESCS_DEFAULT 4096 +#define ENIC_MAX_WQ_DESCS 16384 +#define ENIC_MIN_RQ_DESCS 64 +#define ENIC_MAX_RQ_DESCS 16384 +#define ENIC_MAX_RQ_DESCS_DEFAULT 4096 +#define ENIC_MAX_CQ_DESCS_DEFAULT (64 * 1024) #define ENIC_MIN_MTU ETH_MIN_MTU #define ENIC_MAX_MTU 9000 diff --git a/drivers/net/ethernet/cisco/enic/enic_rq.c b/drivers/net/ethernet/cisco/enic/enic_rq.c new file mode 100644 index 000000000000..ccbf5c9a21d0 --- /dev/null +++ b/drivers/net/ethernet/cisco/enic/enic_rq.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright 2024 Cisco Systems, Inc. All rights reserved. + +#include <linux/skbuff.h> +#include <linux/if_vlan.h> +#include <net/busy_poll.h> +#include "enic.h" +#include "enic_res.h" +#include "enic_rq.h" +#include "vnic_rq.h" +#include "cq_enet_desc.h" + +#define ENIC_LARGE_PKT_THRESHOLD 1000 + +static void enic_intr_update_pkt_size(struct vnic_rx_bytes_counter *pkt_size, + u32 pkt_len) +{ + if (pkt_len > ENIC_LARGE_PKT_THRESHOLD) + pkt_size->large_pkt_bytes_cnt += pkt_len; + else + pkt_size->small_pkt_bytes_cnt += pkt_len; +} + +static void enic_rq_cq_desc_dec(void *cq_desc, u8 cq_desc_size, u8 *type, + u8 *color, u16 *q_number, u16 *completed_index) +{ + /* type_color is the last field for all cq structs */ + u8 type_color; + + switch (cq_desc_size) { + case VNIC_RQ_CQ_ENTRY_SIZE_16: { + struct cq_enet_rq_desc *desc = + (struct cq_enet_rq_desc *)cq_desc; + type_color = desc->type_color; + + /* Make sure color bit is read from desc *before* other fields + * are read from desc. Hardware guarantees color bit is last + * bit (byte) written. Adding the rmb() prevents the compiler + * and/or CPU from reordering the reads which would potentially + * result in reading stale values. + */ + rmb(); + + *q_number = le16_to_cpu(desc->q_number_rss_type_flags) & + CQ_DESC_Q_NUM_MASK; + *completed_index = le16_to_cpu(desc->completed_index_flags) & + CQ_DESC_COMP_NDX_MASK; + break; + } + case VNIC_RQ_CQ_ENTRY_SIZE_32: { + struct cq_enet_rq_desc_32 *desc = + (struct cq_enet_rq_desc_32 *)cq_desc; + type_color = desc->type_color; + + /* Make sure color bit is read from desc *before* other fields + * are read from desc. Hardware guarantees color bit is last + * bit (byte) written. Adding the rmb() prevents the compiler + * and/or CPU from reordering the reads which would potentially + * result in reading stale values. + */ + rmb(); + + *q_number = le16_to_cpu(desc->q_number_rss_type_flags) & + CQ_DESC_Q_NUM_MASK; + *completed_index = le16_to_cpu(desc->completed_index_flags) & + CQ_DESC_COMP_NDX_MASK; + *completed_index |= (desc->fetch_index_flags & CQ_DESC_32_FI_MASK) << + CQ_DESC_COMP_NDX_BITS; + break; + } + case VNIC_RQ_CQ_ENTRY_SIZE_64: { + struct cq_enet_rq_desc_64 *desc = + (struct cq_enet_rq_desc_64 *)cq_desc; + type_color = desc->type_color; + + /* Make sure color bit is read from desc *before* other fields + * are read from desc. Hardware guarantees color bit is last + * bit (byte) written. Adding the rmb() prevents the compiler + * and/or CPU from reordering the reads which would potentially + * result in reading stale values. + */ + rmb(); + + *q_number = le16_to_cpu(desc->q_number_rss_type_flags) & + CQ_DESC_Q_NUM_MASK; + *completed_index = le16_to_cpu(desc->completed_index_flags) & + CQ_DESC_COMP_NDX_MASK; + *completed_index |= (desc->fetch_index_flags & CQ_DESC_64_FI_MASK) << + CQ_DESC_COMP_NDX_BITS; + break; + } + } + + *color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK; + *type = type_color & CQ_DESC_TYPE_MASK; +} + +static void enic_rq_set_skb_flags(struct vnic_rq *vrq, u8 type, u32 rss_hash, + u8 rss_type, u8 fcoe, u8 fcoe_fc_crc_ok, + u8 vlan_stripped, u8 csum_not_calc, + u8 tcp_udp_csum_ok, u8 ipv6, u8 ipv4_csum_ok, + u16 vlan_tci, struct sk_buff *skb) +{ + struct enic *enic = vnic_dev_priv(vrq->vdev); + struct net_device *netdev = enic->netdev; + struct enic_rq_stats *rqstats = &enic->rq[vrq->index].stats; + bool outer_csum_ok = true, encap = false; + + if ((netdev->features & NETIF_F_RXHASH) && rss_hash && type == 3) { + switch (rss_type) { + case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv4: + case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6: + case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6_EX: + skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L4); + rqstats->l4_rss_hash++; + break; + case CQ_ENET_RQ_DESC_RSS_TYPE_IPv4: + case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6: + case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6_EX: + skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L3); + rqstats->l3_rss_hash++; + break; + } + } + if (enic->vxlan.vxlan_udp_port_number) { + switch (enic->vxlan.patch_level) { + case 0: + if (fcoe) { + encap = true; + outer_csum_ok = fcoe_fc_crc_ok; + } + break; + case 2: + if (type == 7 && (rss_hash & BIT(0))) { + encap = true; + outer_csum_ok = (rss_hash & BIT(1)) && + (rss_hash & BIT(2)); + } + break; + } + } + + /* Hardware does not provide whole packet checksum. It only + * provides pseudo checksum. Since hw validates the packet + * checksum but not provide us the checksum value. use + * CHECSUM_UNNECESSARY. + * + * In case of encap pkt tcp_udp_csum_ok/tcp_udp_csum_ok is + * inner csum_ok. outer_csum_ok is set by hw when outer udp + * csum is correct or is zero. + */ + if ((netdev->features & NETIF_F_RXCSUM) && !csum_not_calc && + tcp_udp_csum_ok && outer_csum_ok && (ipv4_csum_ok || ipv6)) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + skb->csum_level = encap; + if (encap) + rqstats->csum_unnecessary_encap++; + else + rqstats->csum_unnecessary++; + } + + if (vlan_stripped) { + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci); + rqstats->vlan_stripped++; + } +} + +/* + * cq_enet_rq_desc accesses section uses only the 1st 15 bytes of the cq which + * is identical for all type (16,32 and 64 byte) of cqs. + */ +static void cq_enet_rq_desc_dec(struct cq_enet_rq_desc *desc, u8 *ingress_port, + u8 *fcoe, u8 *eop, u8 *sop, u8 *rss_type, + u8 *csum_not_calc, u32 *rss_hash, + u16 *bytes_written, u8 *packet_error, + u8 *vlan_stripped, u16 *vlan_tci, + u16 *checksum, u8 *fcoe_sof, + u8 *fcoe_fc_crc_ok, u8 *fcoe_enc_error, + u8 *fcoe_eof, u8 *tcp_udp_csum_ok, u8 *udp, + u8 *tcp, u8 *ipv4_csum_ok, u8 *ipv6, u8 *ipv4, + u8 *ipv4_fragment, u8 *fcs_ok) +{ + u16 completed_index_flags; + u16 q_number_rss_type_flags; + u16 bytes_written_flags; + + completed_index_flags = le16_to_cpu(desc->completed_index_flags); + q_number_rss_type_flags = + le16_to_cpu(desc->q_number_rss_type_flags); + bytes_written_flags = le16_to_cpu(desc->bytes_written_flags); + + *ingress_port = (completed_index_flags & + CQ_ENET_RQ_DESC_FLAGS_INGRESS_PORT) ? 1 : 0; + *fcoe = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_FCOE) ? + 1 : 0; + *eop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_EOP) ? + 1 : 0; + *sop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_SOP) ? + 1 : 0; + + *rss_type = (u8)((q_number_rss_type_flags >> CQ_DESC_Q_NUM_BITS) & + CQ_ENET_RQ_DESC_RSS_TYPE_MASK); + *csum_not_calc = (q_number_rss_type_flags & + CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ? 1 : 0; + + *rss_hash = le32_to_cpu(desc->rss_hash); + + *bytes_written = bytes_written_flags & + CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; + *packet_error = (bytes_written_flags & + CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ? 1 : 0; + *vlan_stripped = (bytes_written_flags & + CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) ? 1 : 0; + + /* + * Tag Control Information(16) = user_priority(3) + cfi(1) + vlan(12) + */ + *vlan_tci = le16_to_cpu(desc->vlan); + + if (*fcoe) { + *fcoe_sof = (u8)(le16_to_cpu(desc->checksum_fcoe) & + CQ_ENET_RQ_DESC_FCOE_SOF_MASK); + *fcoe_fc_crc_ok = (desc->flags & + CQ_ENET_RQ_DESC_FCOE_FC_CRC_OK) ? 1 : 0; + *fcoe_enc_error = (desc->flags & + CQ_ENET_RQ_DESC_FCOE_ENC_ERROR) ? 1 : 0; + *fcoe_eof = (u8)((le16_to_cpu(desc->checksum_fcoe) >> + CQ_ENET_RQ_DESC_FCOE_EOF_SHIFT) & + CQ_ENET_RQ_DESC_FCOE_EOF_MASK); + *checksum = 0; + } else { + *fcoe_sof = 0; + *fcoe_fc_crc_ok = 0; + *fcoe_enc_error = 0; + *fcoe_eof = 0; + *checksum = le16_to_cpu(desc->checksum_fcoe); + } + + *tcp_udp_csum_ok = + (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ? 1 : 0; + *udp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_UDP) ? 1 : 0; + *tcp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP) ? 1 : 0; + *ipv4_csum_ok = + (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ? 1 : 0; + *ipv6 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV6) ? 1 : 0; + *ipv4 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4) ? 1 : 0; + *ipv4_fragment = + (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT) ? 1 : 0; + *fcs_ok = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) ? 1 : 0; +} + +static bool enic_rq_pkt_error(struct vnic_rq *vrq, u8 packet_error, u8 fcs_ok, + u16 bytes_written) +{ + struct enic *enic = vnic_dev_priv(vrq->vdev); + struct enic_rq_stats *rqstats = &enic->rq[vrq->index].stats; + + if (packet_error) { + if (!fcs_ok) { + if (bytes_written > 0) + rqstats->bad_fcs++; + else if (bytes_written == 0) + rqstats->pkt_truncated++; + } + return true; + } + return false; +} + +int enic_rq_alloc_buf(struct vnic_rq *rq) +{ + struct enic *enic = vnic_dev_priv(rq->vdev); + struct net_device *netdev = enic->netdev; + struct enic_rq *erq = &enic->rq[rq->index]; + struct enic_rq_stats *rqstats = &erq->stats; + unsigned int offset = 0; + unsigned int len = netdev->mtu + VLAN_ETH_HLEN; + unsigned int os_buf_index = 0; + dma_addr_t dma_addr; + struct vnic_rq_buf *buf = rq->to_use; + struct page *page; + unsigned int truesize = len; + + if (buf->os_buf) { + enic_queue_rq_desc(rq, buf->os_buf, os_buf_index, buf->dma_addr, + buf->len); + + return 0; + } + + page = page_pool_dev_alloc(erq->pool, &offset, &truesize); + if (unlikely(!page)) { + rqstats->pp_alloc_fail++; + return -ENOMEM; + } + buf->offset = offset; + buf->truesize = truesize; + dma_addr = page_pool_get_dma_addr(page) + offset; + enic_queue_rq_desc(rq, (void *)page, os_buf_index, dma_addr, len); + + return 0; +} + +void enic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf) +{ + struct enic *enic = vnic_dev_priv(rq->vdev); + struct enic_rq *erq = &enic->rq[rq->index]; + + if (!buf->os_buf) + return; + + page_pool_put_full_page(erq->pool, (struct page *)buf->os_buf, true); + buf->os_buf = NULL; +} + +static void enic_rq_indicate_buf(struct enic *enic, struct vnic_rq *rq, + struct vnic_rq_buf *buf, void *cq_desc, + u8 type, u16 q_number, u16 completed_index) +{ + struct sk_buff *skb; + struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)]; + struct enic_rq_stats *rqstats = &enic->rq[rq->index].stats; + struct napi_struct *napi; + + u8 eop, sop, ingress_port, vlan_stripped; + u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof; + u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok; + u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc; + u8 packet_error; + u16 bytes_written, vlan_tci, checksum; + u32 rss_hash; + + rqstats->packets++; + + cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, &ingress_port, + &fcoe, &eop, &sop, &rss_type, &csum_not_calc, + &rss_hash, &bytes_written, &packet_error, + &vlan_stripped, &vlan_tci, &checksum, &fcoe_sof, + &fcoe_fc_crc_ok, &fcoe_enc_error, &fcoe_eof, + &tcp_udp_csum_ok, &udp, &tcp, &ipv4_csum_ok, &ipv6, + &ipv4, &ipv4_fragment, &fcs_ok); + + if (enic_rq_pkt_error(rq, packet_error, fcs_ok, bytes_written)) + return; + + if (eop && bytes_written > 0) { + /* Good receive + */ + rqstats->bytes += bytes_written; + napi = &enic->napi[rq->index]; + skb = napi_get_frags(napi); + if (unlikely(!skb)) { + net_warn_ratelimited("%s: skb alloc error rq[%d], desc[%d]\n", + enic->netdev->name, rq->index, + completed_index); + rqstats->no_skb++; + return; + } + + prefetch(skb->data - NET_IP_ALIGN); + + dma_sync_single_for_cpu(&enic->pdev->dev, buf->dma_addr, + bytes_written, DMA_FROM_DEVICE); + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, + (struct page *)buf->os_buf, buf->offset, + bytes_written, buf->truesize); + skb_record_rx_queue(skb, q_number); + enic_rq_set_skb_flags(rq, type, rss_hash, rss_type, fcoe, + fcoe_fc_crc_ok, vlan_stripped, + csum_not_calc, tcp_udp_csum_ok, ipv6, + ipv4_csum_ok, vlan_tci, skb); + skb_mark_for_recycle(skb); + napi_gro_frags(napi); + if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce) + enic_intr_update_pkt_size(&cq->pkt_size_counter, + bytes_written); + buf->os_buf = NULL; + buf->dma_addr = 0; + buf = buf->next; + } else { + /* Buffer overflow + */ + rqstats->pkt_truncated++; + } +} + +static void enic_rq_service(struct enic *enic, void *cq_desc, u8 type, + u16 q_number, u16 completed_index) +{ + struct enic_rq_stats *rqstats = &enic->rq[q_number].stats; + struct vnic_rq *vrq = &enic->rq[q_number].vrq; + struct vnic_rq_buf *vrq_buf = vrq->to_clean; + int skipped; + + while (1) { + skipped = (vrq_buf->index != completed_index); + if (!skipped) + enic_rq_indicate_buf(enic, vrq, vrq_buf, cq_desc, type, + q_number, completed_index); + else + rqstats->desc_skip++; + + vrq->ring.desc_avail++; + vrq->to_clean = vrq_buf->next; + vrq_buf = vrq_buf->next; + if (!skipped) + break; + } +} + +unsigned int enic_rq_cq_service(struct enic *enic, unsigned int cq_index, + unsigned int work_to_do) +{ + struct vnic_cq *cq = &enic->cq[cq_index]; + void *cq_desc = vnic_cq_to_clean(cq); + u16 q_number, completed_index; + unsigned int work_done = 0; + u8 type, color; + + enic_rq_cq_desc_dec(cq_desc, enic->ext_cq, &type, &color, &q_number, + &completed_index); + + while (color != cq->last_color) { + enic_rq_service(enic, cq_desc, type, q_number, completed_index); + vnic_cq_inc_to_clean(cq); + + if (++work_done >= work_to_do) + break; + + cq_desc = vnic_cq_to_clean(cq); + enic_rq_cq_desc_dec(cq_desc, enic->ext_cq, &type, &color, + &q_number, &completed_index); + } + + return work_done; +} diff --git a/drivers/net/ethernet/cisco/enic/enic_rq.h b/drivers/net/ethernet/cisco/enic/enic_rq.h new file mode 100644 index 000000000000..98476a7297af --- /dev/null +++ b/drivers/net/ethernet/cisco/enic/enic_rq.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright 2024 Cisco Systems, Inc. All rights reserved. + */ + +unsigned int enic_rq_cq_service(struct enic *enic, unsigned int cq_index, + unsigned int work_to_do); +int enic_rq_alloc_buf(struct vnic_rq *rq); +void enic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf); diff --git a/drivers/net/ethernet/cisco/enic/enic_wq.c b/drivers/net/ethernet/cisco/enic/enic_wq.c new file mode 100644 index 000000000000..07936f8b4231 --- /dev/null +++ b/drivers/net/ethernet/cisco/enic/enic_wq.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright 2025 Cisco Systems, Inc. All rights reserved. + +#include <net/netdev_queues.h> +#include "enic_res.h" +#include "enic.h" +#include "enic_wq.h" + +#define ENET_CQ_DESC_COMP_NDX_BITS 14 +#define ENET_CQ_DESC_COMP_NDX_MASK GENMASK(ENET_CQ_DESC_COMP_NDX_BITS - 1, 0) + +static void enic_wq_cq_desc_dec(const struct cq_desc *desc_arg, bool ext_wq, + u8 *type, u8 *color, u16 *q_number, + u16 *completed_index) +{ + const struct cq_desc *desc = desc_arg; + const u8 type_color = desc->type_color; + + *color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK; + + /* + * Make sure color bit is read from desc *before* other fields + * are read from desc. Hardware guarantees color bit is last + * bit (byte) written. Adding the rmb() prevents the compiler + * and/or CPU from reordering the reads which would potentially + * result in reading stale values. + */ + rmb(); + + *type = type_color & CQ_DESC_TYPE_MASK; + *q_number = le16_to_cpu(desc->q_number) & CQ_DESC_Q_NUM_MASK; + + if (ext_wq) + *completed_index = le16_to_cpu(desc->completed_index) & + ENET_CQ_DESC_COMP_NDX_MASK; + else + *completed_index = le16_to_cpu(desc->completed_index) & + CQ_DESC_COMP_NDX_MASK; +} + +void enic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf) +{ + struct enic *enic = vnic_dev_priv(wq->vdev); + + if (buf->sop) + dma_unmap_single(&enic->pdev->dev, buf->dma_addr, buf->len, + DMA_TO_DEVICE); + else + dma_unmap_page(&enic->pdev->dev, buf->dma_addr, buf->len, + DMA_TO_DEVICE); + + if (buf->os_buf) + dev_kfree_skb_any(buf->os_buf); +} + +static void enic_wq_free_buf(struct vnic_wq *wq, struct cq_desc *cq_desc, + struct vnic_wq_buf *buf, void *opaque) +{ + struct enic *enic = vnic_dev_priv(wq->vdev); + + enic->wq[wq->index].stats.cq_work++; + enic->wq[wq->index].stats.cq_bytes += buf->len; + enic_free_wq_buf(wq, buf); +} + +static void enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, + u8 type, u16 q_number, u16 completed_index) +{ + struct enic *enic = vnic_dev_priv(vdev); + + spin_lock(&enic->wq[q_number].lock); + + vnic_wq_service(&enic->wq[q_number].vwq, cq_desc, + completed_index, enic_wq_free_buf, NULL); + + if (netif_tx_queue_stopped(netdev_get_tx_queue(enic->netdev, q_number)) + && vnic_wq_desc_avail(&enic->wq[q_number].vwq) >= + (MAX_SKB_FRAGS + ENIC_DESC_MAX_SPLITS)) { + netif_wake_subqueue(enic->netdev, q_number); + enic->wq[q_number].stats.wake++; + } + + spin_unlock(&enic->wq[q_number].lock); +} + +unsigned int enic_wq_cq_service(struct enic *enic, unsigned int cq_index, + unsigned int work_to_do) +{ + struct vnic_cq *cq = &enic->cq[cq_index]; + u16 q_number, completed_index; + unsigned int work_done = 0; + struct cq_desc *cq_desc; + u8 type, color; + bool ext_wq; + + ext_wq = cq->ring.size > ENIC_MAX_WQ_DESCS_DEFAULT; + + cq_desc = (struct cq_desc *)vnic_cq_to_clean(cq); + enic_wq_cq_desc_dec(cq_desc, ext_wq, &type, &color, + &q_number, &completed_index); + + while (color != cq->last_color) { + enic_wq_service(cq->vdev, cq_desc, type, q_number, + completed_index); + + vnic_cq_inc_to_clean(cq); + + if (++work_done >= work_to_do) + break; + + cq_desc = (struct cq_desc *)vnic_cq_to_clean(cq); + enic_wq_cq_desc_dec(cq_desc, ext_wq, &type, &color, + &q_number, &completed_index); + } + + return work_done; +} diff --git a/drivers/net/ethernet/cisco/enic/enic_wq.h b/drivers/net/ethernet/cisco/enic/enic_wq.h new file mode 100644 index 000000000000..12acb3f2fbc9 --- /dev/null +++ b/drivers/net/ethernet/cisco/enic/enic_wq.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright 2025 Cisco Systems, Inc. All rights reserved. + */ + +void enic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf); +unsigned int enic_wq_cq_service(struct enic *enic, unsigned int cq_index, + unsigned int work_to_do); diff --git a/drivers/net/ethernet/cisco/enic/vnic_cq.h b/drivers/net/ethernet/cisco/enic/vnic_cq.h index eed5bf59e5d2..0e37f5d5e527 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_cq.h +++ b/drivers/net/ethernet/cisco/enic/vnic_cq.h @@ -56,45 +56,18 @@ struct vnic_cq { ktime_t prev_ts; }; -static inline unsigned int vnic_cq_service(struct vnic_cq *cq, - unsigned int work_to_do, - int (*q_service)(struct vnic_dev *vdev, struct cq_desc *cq_desc, - u8 type, u16 q_number, u16 completed_index, void *opaque), - void *opaque) +static inline void *vnic_cq_to_clean(struct vnic_cq *cq) { - struct cq_desc *cq_desc; - unsigned int work_done = 0; - u16 q_number, completed_index; - u8 type, color; - - cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + - cq->ring.desc_size * cq->to_clean); - cq_desc_dec(cq_desc, &type, &color, - &q_number, &completed_index); - - while (color != cq->last_color) { - - if ((*q_service)(cq->vdev, cq_desc, type, - q_number, completed_index, opaque)) - break; - - cq->to_clean++; - if (cq->to_clean == cq->ring.desc_count) { - cq->to_clean = 0; - cq->last_color = cq->last_color ? 0 : 1; - } - - cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + - cq->ring.desc_size * cq->to_clean); - cq_desc_dec(cq_desc, &type, &color, - &q_number, &completed_index); + return ((u8 *)cq->ring.descs + cq->ring.desc_size * cq->to_clean); +} - work_done++; - if (work_done >= work_to_do) - break; +static inline void vnic_cq_inc_to_clean(struct vnic_cq *cq) +{ + cq->to_clean++; + if (cq->to_clean == cq->ring.desc_count) { + cq->to_clean = 0; + cq->last_color = cq->last_color ? 0 : 1; } - - return work_done; } void vnic_cq_free(struct vnic_cq *cq); diff --git a/drivers/net/ethernet/cisco/enic/vnic_devcmd.h b/drivers/net/ethernet/cisco/enic/vnic_devcmd.h index db56d778877a..605ef17f967e 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_devcmd.h +++ b/drivers/net/ethernet/cisco/enic/vnic_devcmd.h @@ -436,6 +436,25 @@ enum vnic_devcmd_cmd { * in: (u16) a2 = unsigned short int port information */ CMD_OVERLAY_OFFLOAD_CFG = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 73), + + /* + * Set extended CQ field in MREGS of RQ (or all RQs) + * for given vNIC + * in: (u64) a0 = RQ selection (VNIC_RQ_ALL for all RQs) + * (u32) a1 = CQ entry size + * VNIC_RQ_CQ_ENTRY_SIZE_16 --> 16 bytes + * VNIC_RQ_CQ_ENTRY_SIZE_32 --> 32 bytes + * VNIC_RQ_CQ_ENTRY_SIZE_64 --> 64 bytes + * + * Capability query: + * out: (u32) a0 = errno, 0:valid cmd + * (u32) a1 = value consisting of supported entries + * bit 0: 16 bytes + * bit 1: 32 bytes + * bit 2: 64 bytes + */ + CMD_CQ_ENTRY_SIZE_SET = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 90), + }; /* CMD_ENABLE2 flags */ diff --git a/drivers/net/ethernet/cisco/enic/vnic_enet.h b/drivers/net/ethernet/cisco/enic/vnic_enet.h index 5acc236069de..9e8e86262a3f 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_enet.h +++ b/drivers/net/ethernet/cisco/enic/vnic_enet.h @@ -21,6 +21,11 @@ struct vnic_enet_config { u16 loop_tag; u16 vf_rq_count; u16 num_arfs; + u8 reserved[66]; + u32 max_rq_ring; // MAX RQ ring size + u32 max_wq_ring; // MAX WQ ring size + u32 max_cq_ring; // MAX CQ ring size + u32 rdma_rsvd_lkey; // Reserved (privileged) LKey }; #define VENETF_TSO 0x1 /* TSO enabled */ diff --git a/drivers/net/ethernet/cisco/enic/vnic_rq.h b/drivers/net/ethernet/cisco/enic/vnic_rq.h index 0bc595abc03b..a1cdd729caec 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_rq.h +++ b/drivers/net/ethernet/cisco/enic/vnic_rq.h @@ -50,7 +50,7 @@ struct vnic_rq_ctrl { (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf)) #define VNIC_RQ_BUF_BLKS_NEEDED(entries) \ DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries)) -#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096) +#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(16384) struct vnic_rq_buf { struct vnic_rq_buf *next; @@ -61,6 +61,8 @@ struct vnic_rq_buf { unsigned int index; void *desc; uint64_t wr_id; + unsigned int offset; + unsigned int truesize; }; enum enic_poll_state { diff --git a/drivers/net/ethernet/cisco/enic/vnic_wq.h b/drivers/net/ethernet/cisco/enic/vnic_wq.h index 75c526911074..3bb4758100ba 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_wq.h +++ b/drivers/net/ethernet/cisco/enic/vnic_wq.h @@ -62,7 +62,7 @@ struct vnic_wq_buf { (VNIC_WQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_wq_buf)) #define VNIC_WQ_BUF_BLKS_NEEDED(entries) \ DIV_ROUND_UP(entries, VNIC_WQ_BUF_BLK_ENTRIES(entries)) -#define VNIC_WQ_BUF_BLKS_MAX VNIC_WQ_BUF_BLKS_NEEDED(4096) +#define VNIC_WQ_BUF_BLKS_MAX VNIC_WQ_BUF_BLKS_NEEDED(16384) struct vnic_wq { unsigned int index; |