diff options
Diffstat (limited to 'drivers/net/ethernet/pensando/ionic/ionic_txrx.c')
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_txrx.c | 945 |
1 files changed, 664 insertions, 281 deletions
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c index 6f4776759863..5dba6d2d633c 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c @@ -5,27 +5,40 @@ #include <linux/ipv6.h> #include <linux/if_vlan.h> #include <net/ip6_checksum.h> +#include <net/netdev_queues.h> #include "ionic.h" #include "ionic_lif.h" #include "ionic_txrx.h" -static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell, - ionic_desc_cb cb_func, void *cb_arg) +static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, + void *data, size_t len); + +static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, + const skb_frag_t *frag, + size_t offset, size_t len); + +static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, + struct ionic_tx_desc_info *desc_info); + +static void ionic_tx_clean(struct ionic_queue *q, + struct ionic_tx_desc_info *desc_info, + struct ionic_txq_comp *comp); + +static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell) { - ionic_q_post(q, ring_dbell, cb_func, cb_arg); + ionic_q_post(q, ring_dbell); } -static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell, - ionic_desc_cb cb_func, void *cb_arg) +static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell) { - ionic_q_post(q, ring_dbell, cb_func, cb_arg); + ionic_q_post(q, ring_dbell); } bool ionic_txq_poke_doorbell(struct ionic_queue *q) { - unsigned long now, then, dif; struct netdev_queue *netdev_txq; + unsigned long now, then, dif; struct net_device *netdev; netdev = q->lif->netdev; @@ -83,46 +96,61 @@ bool ionic_rxq_poke_doorbell(struct ionic_queue *q) return true; } -static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q) +static inline struct ionic_txq_sg_elem *ionic_tx_sg_elems(struct ionic_queue *q) +{ + if (likely(q->sg_desc_size == sizeof(struct ionic_txq_sg_desc_v1))) + return q->txq_sgl_v1[q->head_idx].elems; + else + return q->txq_sgl[q->head_idx].elems; +} + +static inline struct netdev_queue *q_to_ndq(struct net_device *netdev, + struct ionic_queue *q) { - return netdev_get_tx_queue(q->lif->netdev, q->index); + return netdev_get_tx_queue(netdev, q->index); +} + +static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info) +{ + return page_address(buf_info->page) + buf_info->page_offset; +} + +static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info) +{ + return buf_info->dma_addr + buf_info->page_offset; +} + +static unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info) +{ + return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset); } static int ionic_rx_page_alloc(struct ionic_queue *q, struct ionic_buf_info *buf_info) { - struct net_device *netdev = q->lif->netdev; - struct ionic_rx_stats *stats; - struct device *dev; + struct device *dev = q->dev; + dma_addr_t dma_addr; struct page *page; - dev = q->dev; - stats = q_to_rx_stats(q); - - if (unlikely(!buf_info)) { - net_err_ratelimited("%s: %s invalid buf_info in alloc\n", - netdev->name, q->name); - return -EINVAL; - } - page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); if (unlikely(!page)) { net_err_ratelimited("%s: %s page alloc failed\n", - netdev->name, q->name); - stats->alloc_err++; + dev_name(dev), q->name); + q_to_rx_stats(q)->alloc_err++; return -ENOMEM; } - buf_info->dma_addr = dma_map_page(dev, page, 0, - IONIC_PAGE_SIZE, DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) { + dma_addr = dma_map_page(dev, page, 0, + IONIC_PAGE_SIZE, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(dev, dma_addr))) { __free_pages(page, 0); net_err_ratelimited("%s: %s dma map failed\n", - netdev->name, q->name); - stats->dma_map_err++; + dev_name(dev), q->name); + q_to_rx_stats(q)->dma_map_err++; return -EIO; } + buf_info->dma_addr = dma_addr; buf_info->page = page; buf_info->page_offset = 0; @@ -132,12 +160,11 @@ static int ionic_rx_page_alloc(struct ionic_queue *q, static void ionic_rx_page_free(struct ionic_queue *q, struct ionic_buf_info *buf_info) { - struct net_device *netdev = q->lif->netdev; struct device *dev = q->dev; if (unlikely(!buf_info)) { net_err_ratelimited("%s: %s invalid buf_info in free\n", - netdev->name, q->name); + dev_name(dev), q->name); return; } @@ -150,7 +177,7 @@ static void ionic_rx_page_free(struct ionic_queue *q, } static bool ionic_rx_buf_recycle(struct ionic_queue *q, - struct ionic_buf_info *buf_info, u32 used) + struct ionic_buf_info *buf_info, u32 len) { u32 size; @@ -162,7 +189,7 @@ static bool ionic_rx_buf_recycle(struct ionic_queue *q, if (page_to_nid(buf_info->page) != numa_mem_id()) return false; - size = ALIGN(used, IONIC_PAGE_SPLIT_SZ); + size = ALIGN(len, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ); buf_info->page_offset += size; if (buf_info->page_offset >= IONIC_PAGE_SIZE) return false; @@ -172,88 +199,96 @@ static bool ionic_rx_buf_recycle(struct ionic_queue *q, return true; } -static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, - struct ionic_desc_info *desc_info, - struct ionic_rxq_comp *comp) +static void ionic_rx_add_skb_frag(struct ionic_queue *q, + struct sk_buff *skb, + struct ionic_buf_info *buf_info, + u32 off, u32 len, + bool synced) +{ + if (!synced) + dma_sync_single_range_for_cpu(q->dev, ionic_rx_buf_pa(buf_info), + off, len, DMA_FROM_DEVICE); + + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, + buf_info->page, buf_info->page_offset + off, + len, + IONIC_PAGE_SIZE); + + if (!ionic_rx_buf_recycle(q, buf_info, len)) { + dma_unmap_page(q->dev, buf_info->dma_addr, + IONIC_PAGE_SIZE, DMA_FROM_DEVICE); + buf_info->page = NULL; + } +} + +static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q, + struct ionic_rx_desc_info *desc_info, + unsigned int headroom, + unsigned int len, + unsigned int num_sg_elems, + bool synced) { - struct net_device *netdev = q->lif->netdev; struct ionic_buf_info *buf_info; - struct ionic_rx_stats *stats; - struct device *dev = q->dev; struct sk_buff *skb; unsigned int i; u16 frag_len; - u16 len; - - stats = q_to_rx_stats(q); buf_info = &desc_info->bufs[0]; - len = le16_to_cpu(comp->len); - prefetchw(buf_info->page); skb = napi_get_frags(&q_to_qcq(q)->napi); if (unlikely(!skb)) { net_warn_ratelimited("%s: SKB alloc failed on %s!\n", - netdev->name, q->name); - stats->alloc_err++; + dev_name(q->dev), q->name); + q_to_rx_stats(q)->alloc_err++; return NULL; } - i = comp->num_sg_elems + 1; - do { - if (unlikely(!buf_info->page)) { - dev_kfree_skb(skb); - return NULL; - } - - frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN, - IONIC_PAGE_SIZE - buf_info->page_offset)); - len -= frag_len; - - dma_sync_single_for_cpu(dev, - buf_info->dma_addr + buf_info->page_offset, - frag_len, DMA_FROM_DEVICE); - - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - buf_info->page, buf_info->page_offset, frag_len, - IONIC_PAGE_SIZE); - - if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) { - dma_unmap_page(dev, buf_info->dma_addr, - IONIC_PAGE_SIZE, DMA_FROM_DEVICE); - buf_info->page = NULL; - } + if (headroom) + frag_len = min_t(u16, len, + IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); + else + frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); - buf_info++; + if (unlikely(!buf_info->page)) + goto err_bad_buf_page; + ionic_rx_add_skb_frag(q, skb, buf_info, headroom, frag_len, synced); + len -= frag_len; + buf_info++; - i--; - } while (i > 0); + for (i = 0; i < num_sg_elems; i++, buf_info++) { + if (unlikely(!buf_info->page)) + goto err_bad_buf_page; + frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); + ionic_rx_add_skb_frag(q, skb, buf_info, 0, frag_len, synced); + len -= frag_len; + } return skb; + +err_bad_buf_page: + dev_kfree_skb(skb); + return NULL; } -static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, - struct ionic_desc_info *desc_info, - struct ionic_rxq_comp *comp) +static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev, + struct ionic_queue *q, + struct ionic_rx_desc_info *desc_info, + unsigned int headroom, + unsigned int len, + bool synced) { - struct net_device *netdev = q->lif->netdev; struct ionic_buf_info *buf_info; - struct ionic_rx_stats *stats; struct device *dev = q->dev; struct sk_buff *skb; - u16 len; - - stats = q_to_rx_stats(q); buf_info = &desc_info->bufs[0]; - len = le16_to_cpu(comp->len); skb = napi_alloc_skb(&q_to_qcq(q)->napi, len); if (unlikely(!skb)) { net_warn_ratelimited("%s: SKB alloc failed on %s!\n", - netdev->name, q->name); - stats->alloc_err++; + dev_name(dev), q->name); + q_to_rx_stats(q)->alloc_err++; return NULL; } @@ -262,30 +297,343 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, return NULL; } - dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset, - len, DMA_FROM_DEVICE); - skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len); - dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset, - len, DMA_FROM_DEVICE); + if (!synced) + dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), + headroom, len, DMA_FROM_DEVICE); + skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len); + dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info), + headroom, len, DMA_FROM_DEVICE); skb_put(skb, len); - skb->protocol = eth_type_trans(skb, q->lif->netdev); + skb->protocol = eth_type_trans(skb, netdev); return skb; } +static void ionic_xdp_tx_desc_clean(struct ionic_queue *q, + struct ionic_tx_desc_info *desc_info) +{ + unsigned int nbufs = desc_info->nbufs; + struct ionic_buf_info *buf_info; + struct device *dev = q->dev; + int i; + + if (!nbufs) + return; + + buf_info = desc_info->bufs; + dma_unmap_single(dev, buf_info->dma_addr, + buf_info->len, DMA_TO_DEVICE); + if (desc_info->act == XDP_TX) + __free_pages(buf_info->page, 0); + buf_info->page = NULL; + + buf_info++; + for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) { + dma_unmap_page(dev, buf_info->dma_addr, + buf_info->len, DMA_TO_DEVICE); + if (desc_info->act == XDP_TX) + __free_pages(buf_info->page, 0); + buf_info->page = NULL; + } + + if (desc_info->act == XDP_REDIRECT) + xdp_return_frame(desc_info->xdpf); + + desc_info->nbufs = 0; + desc_info->xdpf = NULL; + desc_info->act = 0; +} + +static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame, + enum xdp_action act, struct page *page, int off, + bool ring_doorbell) +{ + struct ionic_tx_desc_info *desc_info; + struct ionic_buf_info *buf_info; + struct ionic_tx_stats *stats; + struct ionic_txq_desc *desc; + size_t len = frame->len; + dma_addr_t dma_addr; + u64 cmd; + + desc_info = &q->tx_info[q->head_idx]; + desc = &q->txq[q->head_idx]; + buf_info = desc_info->bufs; + stats = q_to_tx_stats(q); + + dma_addr = ionic_tx_map_single(q, frame->data, len); + if (!dma_addr) + return -EIO; + buf_info->dma_addr = dma_addr; + buf_info->len = len; + buf_info->page = page; + buf_info->page_offset = off; + + desc_info->nbufs = 1; + desc_info->xdpf = frame; + desc_info->act = act; + + if (xdp_frame_has_frags(frame)) { + struct ionic_txq_sg_elem *elem; + struct skb_shared_info *sinfo; + struct ionic_buf_info *bi; + skb_frag_t *frag; + int i; + + bi = &buf_info[1]; + sinfo = xdp_get_shared_info_from_frame(frame); + frag = sinfo->frags; + elem = ionic_tx_sg_elems(q); + for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) { + dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); + if (!dma_addr) { + ionic_tx_desc_unmap_bufs(q, desc_info); + return -EIO; + } + bi->dma_addr = dma_addr; + bi->len = skb_frag_size(frag); + bi->page = skb_frag_page(frag); + + elem->addr = cpu_to_le64(bi->dma_addr); + elem->len = cpu_to_le16(bi->len); + elem++; + + desc_info->nbufs++; + } + } + + cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, + 0, (desc_info->nbufs - 1), buf_info->dma_addr); + desc->cmd = cpu_to_le64(cmd); + desc->len = cpu_to_le16(len); + desc->csum_start = 0; + desc->csum_offset = 0; + + stats->xdp_frames++; + stats->pkts++; + stats->bytes += len; + + ionic_txq_post(q, ring_doorbell); + + return 0; +} + +int ionic_xdp_xmit(struct net_device *netdev, int n, + struct xdp_frame **xdp_frames, u32 flags) +{ + struct ionic_lif *lif = netdev_priv(netdev); + struct ionic_queue *txq; + struct netdev_queue *nq; + int nxmit; + int space; + int cpu; + int qi; + + if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) + return -ENETDOWN; + + if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) + return -EINVAL; + + /* AdminQ is assumed on cpu 0, while we attempt to affinitize the + * TxRx queue pairs 0..n-1 on cpus 1..n. We try to keep with that + * affinitization here, but of course irqbalance and friends might + * have juggled things anyway, so we have to check for the 0 case. + */ + cpu = smp_processor_id(); + qi = cpu ? (cpu - 1) % lif->nxqs : cpu; + + txq = &lif->txqcqs[qi]->q; + nq = netdev_get_tx_queue(netdev, txq->index); + __netif_tx_lock(nq, cpu); + txq_trans_cond_update(nq); + + if (netif_tx_queue_stopped(nq) || + !netif_txq_maybe_stop(q_to_ndq(netdev, txq), + ionic_q_space_avail(txq), + 1, 1)) { + __netif_tx_unlock(nq); + return -EIO; + } + + space = min_t(int, n, ionic_q_space_avail(txq)); + for (nxmit = 0; nxmit < space ; nxmit++) { + if (ionic_xdp_post_frame(txq, xdp_frames[nxmit], + XDP_REDIRECT, + virt_to_page(xdp_frames[nxmit]->data), + 0, false)) { + nxmit--; + break; + } + } + + if (flags & XDP_XMIT_FLUSH) + ionic_dbell_ring(lif->kern_dbpage, txq->hw_type, + txq->dbval | txq->head_idx); + + netif_txq_maybe_stop(q_to_ndq(netdev, txq), + ionic_q_space_avail(txq), + 4, 4); + __netif_tx_unlock(nq); + + return nxmit; +} + +static bool ionic_run_xdp(struct ionic_rx_stats *stats, + struct net_device *netdev, + struct bpf_prog *xdp_prog, + struct ionic_queue *rxq, + struct ionic_buf_info *buf_info, + int len) +{ + u32 xdp_action = XDP_ABORTED; + struct xdp_buff xdp_buf; + struct ionic_queue *txq; + struct netdev_queue *nq; + struct xdp_frame *xdpf; + int remain_len; + int frag_len; + int err = 0; + + xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info); + frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); + xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info), + XDP_PACKET_HEADROOM, frag_len, false); + + dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info), + XDP_PACKET_HEADROOM, len, + DMA_FROM_DEVICE); + + prefetchw(&xdp_buf.data_hard_start); + + /* We limit MTU size to one buffer if !xdp_has_frags, so + * if the recv len is bigger than one buffer + * then we know we have frag info to gather + */ + remain_len = len - frag_len; + if (remain_len) { + struct skb_shared_info *sinfo; + struct ionic_buf_info *bi; + skb_frag_t *frag; + + bi = buf_info; + sinfo = xdp_get_shared_info_from_buff(&xdp_buf); + sinfo->nr_frags = 0; + sinfo->xdp_frags_size = 0; + xdp_buff_set_frags_flag(&xdp_buf); + + do { + if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) { + err = -ENOSPC; + goto out_xdp_abort; + } + + frag = &sinfo->frags[sinfo->nr_frags]; + sinfo->nr_frags++; + bi++; + frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi)); + dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi), + 0, frag_len, DMA_FROM_DEVICE); + skb_frag_fill_page_desc(frag, bi->page, 0, frag_len); + sinfo->xdp_frags_size += frag_len; + remain_len -= frag_len; + + if (page_is_pfmemalloc(bi->page)) + xdp_buff_set_frag_pfmemalloc(&xdp_buf); + } while (remain_len > 0); + } + + xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf); + + switch (xdp_action) { + case XDP_PASS: + stats->xdp_pass++; + return false; /* false = we didn't consume the packet */ + + case XDP_DROP: + ionic_rx_page_free(rxq, buf_info); + stats->xdp_drop++; + break; + + case XDP_TX: + xdpf = xdp_convert_buff_to_frame(&xdp_buf); + if (!xdpf) + goto out_xdp_abort; + + txq = rxq->partner; + nq = netdev_get_tx_queue(netdev, txq->index); + __netif_tx_lock(nq, smp_processor_id()); + txq_trans_cond_update(nq); + + if (netif_tx_queue_stopped(nq) || + !netif_txq_maybe_stop(q_to_ndq(netdev, txq), + ionic_q_space_avail(txq), + 1, 1)) { + __netif_tx_unlock(nq); + goto out_xdp_abort; + } + + dma_unmap_page(rxq->dev, buf_info->dma_addr, + IONIC_PAGE_SIZE, DMA_FROM_DEVICE); + + err = ionic_xdp_post_frame(txq, xdpf, XDP_TX, + buf_info->page, + buf_info->page_offset, + true); + __netif_tx_unlock(nq); + if (err) { + netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err); + goto out_xdp_abort; + } + stats->xdp_tx++; + + /* the Tx completion will free the buffers */ + break; + + case XDP_REDIRECT: + /* unmap the pages before handing them to a different device */ + dma_unmap_page(rxq->dev, buf_info->dma_addr, + IONIC_PAGE_SIZE, DMA_FROM_DEVICE); + + err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog); + if (err) { + netdev_dbg(netdev, "xdp_do_redirect err %d\n", err); + goto out_xdp_abort; + } + buf_info->page = NULL; + rxq->xdp_flush = true; + stats->xdp_redirect++; + break; + + case XDP_ABORTED: + default: + goto out_xdp_abort; + } + + return true; + +out_xdp_abort: + trace_xdp_exception(netdev, xdp_prog, xdp_action); + ionic_rx_page_free(rxq, buf_info); + stats->xdp_aborted++; + + return true; +} + static void ionic_rx_clean(struct ionic_queue *q, - struct ionic_desc_info *desc_info, - struct ionic_cq_info *cq_info, - void *cb_arg) + struct ionic_rx_desc_info *desc_info, + struct ionic_rxq_comp *comp) { struct net_device *netdev = q->lif->netdev; struct ionic_qcq *qcq = q_to_qcq(q); struct ionic_rx_stats *stats; - struct ionic_rxq_comp *comp; + struct bpf_prog *xdp_prog; + unsigned int headroom; struct sk_buff *skb; - - comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp); + bool synced = false; + bool use_copybreak; + u16 len; stats = q_to_rx_stats(q); @@ -294,13 +642,25 @@ static void ionic_rx_clean(struct ionic_queue *q, return; } + len = le16_to_cpu(comp->len); stats->pkts++; - stats->bytes += le16_to_cpu(comp->len); + stats->bytes += len; + + xdp_prog = READ_ONCE(q->lif->xdp_prog); + if (xdp_prog) { + if (ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len)) + return; + synced = true; + } - if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) - skb = ionic_rx_copybreak(q, desc_info, comp); + headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; + use_copybreak = len <= q->lif->rx_copybreak; + if (use_copybreak) + skb = ionic_rx_copybreak(netdev, q, desc_info, + headroom, len, synced); else - skb = ionic_rx_frags(q, desc_info, comp); + skb = ionic_rx_build_skb(q, desc_info, headroom, len, + comp->num_sg_elems, synced); if (unlikely(!skb)) { stats->dropped++; @@ -352,7 +712,7 @@ static void ionic_rx_clean(struct ionic_queue *q, u64 hwstamp; cq_desc_hwstamp = - cq_info->cq_desc + + (void *)comp + qcq->cq.desc_size - sizeof(struct ionic_rxq_comp) - IONIC_HWSTAMP_CQ_NEGOFFSET; @@ -367,19 +727,19 @@ static void ionic_rx_clean(struct ionic_queue *q, } } - if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) + if (use_copybreak) napi_gro_receive(&qcq->napi, skb); else napi_gro_frags(&qcq->napi); } -bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) +bool ionic_rx_service(struct ionic_cq *cq) { + struct ionic_rx_desc_info *desc_info; struct ionic_queue *q = cq->bound_q; - struct ionic_desc_info *desc_info; struct ionic_rxq_comp *comp; - comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); + comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx]; if (!color_match(comp->pkt_type_color, cq->done_color)) return false; @@ -391,31 +751,29 @@ bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) if (q->tail_idx != le16_to_cpu(comp->comp_index)) return false; - desc_info = &q->info[q->tail_idx]; + desc_info = &q->rx_info[q->tail_idx]; q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); /* clean the related q entry, only one per qc completion */ - ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg); - - desc_info->cb = NULL; - desc_info->cb_arg = NULL; + ionic_rx_clean(q, desc_info, comp); return true; } static inline void ionic_write_cmb_desc(struct ionic_queue *q, - void __iomem *cmb_desc, void *desc) { - if (q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS) - memcpy_toio(cmb_desc, desc, q->desc_size); + /* Since Rx and Tx descriptors are the same size, we can + * save an instruction or two and skip the qtype check. + */ + if (unlikely(q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS)) + memcpy_toio(&q->cmb_txq[q->head_idx], desc, sizeof(q->cmb_txq[0])); } void ionic_rx_fill(struct ionic_queue *q) { struct net_device *netdev = q->lif->netdev; - struct ionic_desc_info *desc_info; - struct ionic_rxq_sg_desc *sg_desc; + struct ionic_rx_desc_info *desc_info; struct ionic_rxq_sg_elem *sg_elem; struct ionic_buf_info *buf_info; unsigned int fill_threshold; @@ -424,8 +782,9 @@ void ionic_rx_fill(struct ionic_queue *q) unsigned int frag_len; unsigned int nfrags; unsigned int n_fill; - unsigned int i, j; unsigned int len; + unsigned int i; + unsigned int j; n_fill = ionic_q_space_avail(q); @@ -434,13 +793,16 @@ void ionic_rx_fill(struct ionic_queue *q) if (n_fill < fill_threshold) return; - len = netdev->mtu + ETH_HLEN + VLAN_HLEN; + len = netdev->mtu + VLAN_ETH_HLEN; for (i = n_fill; i; i--) { + unsigned int headroom; + unsigned int buf_len; + nfrags = 0; remain_len = len; - desc_info = &q->info[q->head_idx]; - desc = desc_info->desc; + desc = &q->rxq[q->head_idx]; + desc_info = &q->rx_info[q->head_idx]; buf_info = &desc_info->bufs[0]; if (!buf_info->page) { /* alloc a new buffer? */ @@ -451,19 +813,26 @@ void ionic_rx_fill(struct ionic_queue *q) } } - /* fill main descriptor - buf[0] */ - desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); - frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN, - IONIC_PAGE_SIZE - buf_info->page_offset)); + /* fill main descriptor - buf[0] + * XDP uses space in the first buffer, so account for + * head room, tail room, and ip header in the first frag size. + */ + headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; + if (q->xdp_rxq_info) + buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN; + else + buf_len = ionic_rx_buf_size(buf_info); + frag_len = min_t(u16, len, buf_len); + + desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom); desc->len = cpu_to_le16(frag_len); remain_len -= frag_len; buf_info++; nfrags++; /* fill sg descriptors - buf[1..n] */ - sg_desc = desc_info->sg_desc; - for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) { - sg_elem = &sg_desc->elems[j]; + sg_elem = q->rxq_sgl[q->head_idx].elems; + for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++, sg_elem++) { if (!buf_info->page) { /* alloc a new sg buffer? */ if (unlikely(ionic_rx_page_alloc(q, buf_info))) { sg_elem->addr = 0; @@ -472,10 +841,8 @@ void ionic_rx_fill(struct ionic_queue *q) } } - sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); - frag_len = min_t(u16, remain_len, min_t(u32, IONIC_MAX_BUF_LEN, - IONIC_PAGE_SIZE - - buf_info->page_offset)); + sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info)); + frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info)); sg_elem->len = cpu_to_le16(frag_len); remain_len -= frag_len; buf_info++; @@ -483,18 +850,16 @@ void ionic_rx_fill(struct ionic_queue *q) } /* clear end sg element as a sentinel */ - if (j < q->max_sg_elems) { - sg_elem = &sg_desc->elems[j]; + if (j < q->max_sg_elems) memset(sg_elem, 0, sizeof(*sg_elem)); - } desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : IONIC_RXQ_DESC_OPCODE_SIMPLE; desc_info->nbufs = nfrags; - ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); + ionic_write_cmb_desc(q, desc); - ionic_rxq_post(q, false, ionic_rx_clean, NULL); + ionic_rxq_post(q, false); } ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, @@ -509,21 +874,19 @@ void ionic_rx_fill(struct ionic_queue *q) void ionic_rx_empty(struct ionic_queue *q) { - struct ionic_desc_info *desc_info; + struct ionic_rx_desc_info *desc_info; struct ionic_buf_info *buf_info; unsigned int i, j; for (i = 0; i < q->num_descs; i++) { - desc_info = &q->info[i]; - for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) { + desc_info = &q->rx_info[i]; + for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++) { buf_info = &desc_info->bufs[j]; if (buf_info->page) ionic_rx_page_free(q, buf_info); } desc_info->nbufs = 0; - desc_info->cb = NULL; - desc_info->cb_arg = NULL; } q->head_idx = 0; @@ -568,16 +931,10 @@ int ionic_tx_napi(struct napi_struct *napi, int budget) { struct ionic_qcq *qcq = napi_to_qcq(napi); struct ionic_cq *cq = napi_to_cq(napi); - struct ionic_dev *idev; - struct ionic_lif *lif; u32 work_done = 0; u32 flags = 0; - lif = cq->bound_q->lif; - idev = &lif->ionic->idev; - - work_done = ionic_cq_service(cq, budget, - ionic_tx_service, NULL, NULL); + work_done = ionic_tx_cq_service(cq, budget); if (unlikely(!budget)) return budget; @@ -590,7 +947,7 @@ int ionic_tx_napi(struct napi_struct *napi, int budget) if (work_done || flags) { flags |= IONIC_INTR_CRED_RESET_COALESCE; - ionic_intr_credits(idev->intr_ctrl, + ionic_intr_credits(cq->idev->intr_ctrl, cq->bound_intr->index, work_done, flags); } @@ -601,26 +958,30 @@ int ionic_tx_napi(struct napi_struct *napi, int budget) return work_done; } +static void ionic_xdp_do_flush(struct ionic_cq *cq) +{ + if (cq->bound_q->xdp_flush) { + xdp_do_flush(); + cq->bound_q->xdp_flush = false; + } +} + int ionic_rx_napi(struct napi_struct *napi, int budget) { struct ionic_qcq *qcq = napi_to_qcq(napi); struct ionic_cq *cq = napi_to_cq(napi); - struct ionic_dev *idev; - struct ionic_lif *lif; u32 work_done = 0; u32 flags = 0; if (unlikely(!budget)) return budget; - lif = cq->bound_q->lif; - idev = &lif->ionic->idev; - work_done = ionic_cq_service(cq, budget, ionic_rx_service, NULL, NULL); ionic_rx_fill(cq->bound_q); + ionic_xdp_do_flush(cq); if (work_done < budget && napi_complete_done(napi, work_done)) { ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); flags |= IONIC_INTR_CRED_UNMASK; @@ -629,7 +990,7 @@ int ionic_rx_napi(struct napi_struct *napi, int budget) if (work_done || flags) { flags |= IONIC_INTR_CRED_RESET_COALESCE; - ionic_intr_credits(idev->intr_ctrl, + ionic_intr_credits(cq->idev->intr_ctrl, cq->bound_intr->index, work_done, flags); } @@ -646,7 +1007,6 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget) struct ionic_cq *rxcq = napi_to_cq(napi); unsigned int qi = rxcq->bound_q->index; struct ionic_qcq *txqcq; - struct ionic_dev *idev; struct ionic_lif *lif; struct ionic_cq *txcq; bool resched = false; @@ -655,12 +1015,10 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget) u32 flags = 0; lif = rxcq->bound_q->lif; - idev = &lif->ionic->idev; txqcq = lif->txqcqs[qi]; txcq = &lif->txqcqs[qi]->cq; - tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT, - ionic_tx_service, NULL, NULL); + tx_work_done = ionic_tx_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT); if (unlikely(!budget)) return budget; @@ -670,6 +1028,7 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget) ionic_rx_fill(rxcq->bound_q); + ionic_xdp_do_flush(rxcq); if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { ionic_dim_update(rxqcq, 0); flags |= IONIC_INTR_CRED_UNMASK; @@ -678,7 +1037,7 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget) if (rx_work_done || flags) { flags |= IONIC_INTR_CRED_RESET_COALESCE; - ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index, + ionic_intr_credits(rxcq->idev->intr_ctrl, rxcq->bound_intr->index, tx_work_done + rx_work_done, flags); } @@ -695,15 +1054,14 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget) static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, void *data, size_t len) { - struct ionic_tx_stats *stats = q_to_tx_stats(q); struct device *dev = q->dev; dma_addr_t dma_addr; dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma_addr)) { net_warn_ratelimited("%s: DMA single map failed on %s!\n", - q->lif->netdev->name, q->name); - stats->dma_map_err++; + dev_name(dev), q->name); + q_to_tx_stats(q)->dma_map_err++; return 0; } return dma_addr; @@ -713,24 +1071,23 @@ static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, const skb_frag_t *frag, size_t offset, size_t len) { - struct ionic_tx_stats *stats = q_to_tx_stats(q); struct device *dev = q->dev; dma_addr_t dma_addr; dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma_addr)) { net_warn_ratelimited("%s: DMA frag map failed on %s!\n", - q->lif->netdev->name, q->name); - stats->dma_map_err++; + dev_name(dev), q->name); + q_to_tx_stats(q)->dma_map_err++; + return 0; } return dma_addr; } static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, - struct ionic_desc_info *desc_info) + struct ionic_tx_desc_info *desc_info) { struct ionic_buf_info *buf_info = desc_info->bufs; - struct ionic_tx_stats *stats = q_to_tx_stats(q); struct device *dev = q->dev; dma_addr_t dma_addr; unsigned int nfrags; @@ -738,10 +1095,8 @@ static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, int frag_idx; dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); - if (dma_mapping_error(dev, dma_addr)) { - stats->dma_map_err++; + if (!dma_addr) return -EIO; - } buf_info->dma_addr = dma_addr; buf_info->len = skb_headlen(skb); buf_info++; @@ -750,10 +1105,8 @@ static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, nfrags = skb_shinfo(skb)->nr_frags; for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) { dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); - if (dma_mapping_error(dev, dma_addr)) { - stats->dma_map_err++; + if (!dma_addr) goto dma_fail; - } buf_info->dma_addr = dma_addr; buf_info->len = skb_frag_size(frag); buf_info++; @@ -771,12 +1124,13 @@ dma_fail: dma_unmap_page(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE); } - dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE); + dma_unmap_single(dev, desc_info->bufs[0].dma_addr, + desc_info->bufs[0].len, DMA_TO_DEVICE); return -EIO; } static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, - struct ionic_desc_info *desc_info) + struct ionic_tx_desc_info *desc_info) { struct ionic_buf_info *buf_info = desc_info->bufs; struct device *dev = q->dev; @@ -785,41 +1139,48 @@ static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, if (!desc_info->nbufs) return; - dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr, + dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE); buf_info++; for (i = 1; i < desc_info->nbufs; i++, buf_info++) - dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr, + dma_unmap_page(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE); desc_info->nbufs = 0; } static void ionic_tx_clean(struct ionic_queue *q, - struct ionic_desc_info *desc_info, - struct ionic_cq_info *cq_info, - void *cb_arg) + struct ionic_tx_desc_info *desc_info, + struct ionic_txq_comp *comp) { struct ionic_tx_stats *stats = q_to_tx_stats(q); struct ionic_qcq *qcq = q_to_qcq(q); - struct sk_buff *skb = cb_arg; - u16 qi; + struct sk_buff *skb; + + if (desc_info->xdpf) { + ionic_xdp_tx_desc_clean(q->partner, desc_info); + stats->clean++; + + if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index))) + netif_wake_subqueue(q->lif->netdev, q->index); + + return; + } ionic_tx_desc_unmap_bufs(q, desc_info); + skb = desc_info->skb; if (!skb) return; - qi = skb_get_queue_mapping(skb); - - if (ionic_txq_hwstamp_enabled(q)) { - if (cq_info) { + if (unlikely(ionic_txq_hwstamp_enabled(q))) { + if (comp) { struct skb_shared_hwtstamps hwts = {}; __le64 *cq_desc_hwstamp; u64 hwstamp; cq_desc_hwstamp = - cq_info->cq_desc + + (void *)comp + qcq->cq.desc_size - sizeof(struct ionic_txq_comp) - IONIC_HWSTAMP_CQ_NEGOFFSET; @@ -837,27 +1198,25 @@ static void ionic_tx_clean(struct ionic_queue *q, stats->hwstamp_invalid++; } } - - } else if (unlikely(__netif_subqueue_stopped(q->lif->netdev, qi))) { - netif_wake_subqueue(q->lif->netdev, qi); } desc_info->bytes = skb->len; stats->clean++; - dev_consume_skb_any(skb); + napi_consume_skb(skb, 1); } -bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) +static bool ionic_tx_service(struct ionic_cq *cq, + unsigned int *total_pkts, unsigned int *total_bytes) { + struct ionic_tx_desc_info *desc_info; struct ionic_queue *q = cq->bound_q; - struct ionic_desc_info *desc_info; struct ionic_txq_comp *comp; - int bytes = 0; - int pkts = 0; + unsigned int bytes = 0; + unsigned int pkts = 0; u16 index; - comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); + comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx]; if (!color_match(comp->color, cq->done_color)) return false; @@ -866,59 +1225,90 @@ bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) * several q entries completed for each cq completion */ do { - desc_info = &q->info[q->tail_idx]; + desc_info = &q->tx_info[q->tail_idx]; desc_info->bytes = 0; index = q->tail_idx; q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); - ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg); - if (desc_info->cb_arg) { + ionic_tx_clean(q, desc_info, comp); + if (desc_info->skb) { pkts++; bytes += desc_info->bytes; + desc_info->skb = NULL; } - desc_info->cb = NULL; - desc_info->cb_arg = NULL; } while (index != le16_to_cpu(comp->comp_index)); - if (pkts && bytes && !ionic_txq_hwstamp_enabled(q)) - netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); + (*total_pkts) += pkts; + (*total_bytes) += bytes; return true; } +unsigned int ionic_tx_cq_service(struct ionic_cq *cq, unsigned int work_to_do) +{ + unsigned int work_done = 0; + unsigned int bytes = 0; + unsigned int pkts = 0; + + if (work_to_do == 0) + return 0; + + while (ionic_tx_service(cq, &pkts, &bytes)) { + if (cq->tail_idx == cq->num_descs - 1) + cq->done_color = !cq->done_color; + cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); + + if (++work_done >= work_to_do) + break; + } + + if (work_done) { + struct ionic_queue *q = cq->bound_q; + + if (likely(!ionic_txq_hwstamp_enabled(q))) + netif_txq_completed_wake(q_to_ndq(q->lif->netdev, q), + pkts, bytes, + ionic_q_space_avail(q), + IONIC_TSO_DESCS_NEEDED); + } + + return work_done; +} + void ionic_tx_flush(struct ionic_cq *cq) { - struct ionic_dev *idev = &cq->lif->ionic->idev; u32 work_done; - work_done = ionic_cq_service(cq, cq->num_descs, - ionic_tx_service, NULL, NULL); + work_done = ionic_tx_cq_service(cq, cq->num_descs); if (work_done) - ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index, + ionic_intr_credits(cq->idev->intr_ctrl, cq->bound_intr->index, work_done, IONIC_INTR_CRED_RESET_COALESCE); } void ionic_tx_empty(struct ionic_queue *q) { - struct ionic_desc_info *desc_info; + struct ionic_tx_desc_info *desc_info; int bytes = 0; int pkts = 0; /* walk the not completed tx entries, if any */ while (q->head_idx != q->tail_idx) { - desc_info = &q->info[q->tail_idx]; + desc_info = &q->tx_info[q->tail_idx]; desc_info->bytes = 0; q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); - ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg); - if (desc_info->cb_arg) { + ionic_tx_clean(q, desc_info, NULL); + if (desc_info->skb) { pkts++; bytes += desc_info->bytes; + desc_info->skb = NULL; } - desc_info->cb = NULL; - desc_info->cb_arg = NULL; } - if (pkts && bytes && !ionic_txq_hwstamp_enabled(q)) - netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); + if (likely(!ionic_txq_hwstamp_enabled(q))) { + struct netdev_queue *ndq = q_to_ndq(q->lif->netdev, q); + + netdev_tx_completed_queue(ndq, pkts, bytes); + netdev_tx_reset_queue(ndq); + } } static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) @@ -966,8 +1356,8 @@ static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb) return 0; } -static void ionic_tx_tso_post(struct ionic_queue *q, - struct ionic_desc_info *desc_info, +static void ionic_tx_tso_post(struct net_device *netdev, struct ionic_queue *q, + struct ionic_tx_desc_info *desc_info, struct sk_buff *skb, dma_addr_t addr, u8 nsge, u16 len, unsigned int hdrlen, unsigned int mss, @@ -975,7 +1365,7 @@ static void ionic_tx_tso_post(struct ionic_queue *q, u16 vlan_tci, bool has_vlan, bool start, bool done) { - struct ionic_txq_desc *desc = desc_info->desc; + struct ionic_txq_desc *desc = &q->txq[q->head_idx]; u8 flags = 0; u64 cmd; @@ -991,22 +1381,23 @@ static void ionic_tx_tso_post(struct ionic_queue *q, desc->hdr_len = cpu_to_le16(hdrlen); desc->mss = cpu_to_le16(mss); - ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); + ionic_write_cmb_desc(q, desc); if (start) { skb_tx_timestamp(skb); - if (!ionic_txq_hwstamp_enabled(q)) - netdev_tx_sent_queue(q_to_ndq(q), skb->len); - ionic_txq_post(q, false, ionic_tx_clean, skb); + if (likely(!ionic_txq_hwstamp_enabled(q))) + netdev_tx_sent_queue(q_to_ndq(netdev, q), skb->len); + ionic_txq_post(q, false); } else { - ionic_txq_post(q, done, NULL, NULL); + ionic_txq_post(q, done); } } -static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) +static int ionic_tx_tso(struct net_device *netdev, struct ionic_queue *q, + struct sk_buff *skb) { struct ionic_tx_stats *stats = q_to_tx_stats(q); - struct ionic_desc_info *desc_info; + struct ionic_tx_desc_info *desc_info; struct ionic_buf_info *buf_info; struct ionic_txq_sg_elem *elem; struct ionic_txq_desc *desc; @@ -1028,8 +1419,7 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) bool encap; int err; - desc_info = &q->info[q->head_idx]; - buf_info = desc_info->bufs; + desc_info = &q->tx_info[q->head_idx]; if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) return -EIO; @@ -1066,6 +1456,8 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) else hdrlen = skb_tcp_all_headers(skb); + desc_info->skb = skb; + buf_info = desc_info->bufs; tso_rem = len; seg_rem = min(tso_rem, hdrlen + mss); @@ -1092,8 +1484,8 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) chunk_len = min(frag_rem, seg_rem); if (!desc) { /* fill main descriptor */ - desc = desc_info->txq_desc; - elem = desc_info->txq_sg_desc->elems; + desc = &q->txq[q->head_idx]; + elem = ionic_tx_sg_elems(q); desc_addr = frag_addr; desc_len = chunk_len; } else { @@ -1111,13 +1503,13 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) seg_rem = min(tso_rem, mss); done = (tso_rem == 0); /* post descriptor */ - ionic_tx_tso_post(q, desc_info, skb, + ionic_tx_tso_post(netdev, q, desc_info, skb, desc_addr, desc_nsge, desc_len, hdrlen, mss, outer_csum, vlan_tci, has_vlan, start, done); start = false; /* Buffer information is stored with the first tso descriptor */ - desc_info = &q->info[q->head_idx]; + desc_info = &q->tx_info[q->head_idx]; desc_info->nbufs = 0; } @@ -1130,9 +1522,9 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) } static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, - struct ionic_desc_info *desc_info) + struct ionic_tx_desc_info *desc_info) { - struct ionic_txq_desc *desc = desc_info->txq_desc; + struct ionic_txq_desc *desc = &q->txq[q->head_idx]; struct ionic_buf_info *buf_info = desc_info->bufs; struct ionic_tx_stats *stats = q_to_tx_stats(q); bool has_vlan; @@ -1160,7 +1552,7 @@ static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); desc->csum_offset = cpu_to_le16(skb->csum_offset); - ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); + ionic_write_cmb_desc(q, desc); if (skb_csum_is_sctp(skb)) stats->crc32_csum++; @@ -1169,9 +1561,9 @@ static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, } static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb, - struct ionic_desc_info *desc_info) + struct ionic_tx_desc_info *desc_info) { - struct ionic_txq_desc *desc = desc_info->txq_desc; + struct ionic_txq_desc *desc = &q->txq[q->head_idx]; struct ionic_buf_info *buf_info = desc_info->bufs; struct ionic_tx_stats *stats = q_to_tx_stats(q); bool has_vlan; @@ -1199,20 +1591,20 @@ static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb, desc->csum_start = 0; desc->csum_offset = 0; - ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); + ionic_write_cmb_desc(q, desc); stats->csum_none++; } static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb, - struct ionic_desc_info *desc_info) + struct ionic_tx_desc_info *desc_info) { - struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc; struct ionic_buf_info *buf_info = &desc_info->bufs[1]; - struct ionic_txq_sg_elem *elem = sg_desc->elems; struct ionic_tx_stats *stats = q_to_tx_stats(q); + struct ionic_txq_sg_elem *elem; unsigned int i; + elem = ionic_tx_sg_elems(q); for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) { elem->addr = cpu_to_le64(buf_info->dma_addr); elem->len = cpu_to_le16(buf_info->len); @@ -1221,14 +1613,18 @@ static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb, stats->frags += skb_shinfo(skb)->nr_frags; } -static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb) +static int ionic_tx(struct net_device *netdev, struct ionic_queue *q, + struct sk_buff *skb) { - struct ionic_desc_info *desc_info = &q->info[q->head_idx]; + struct ionic_tx_desc_info *desc_info = &q->tx_info[q->head_idx]; struct ionic_tx_stats *stats = q_to_tx_stats(q); + bool ring_dbell = true; if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) return -EIO; + desc_info->skb = skb; + /* set up the initial descriptor */ if (skb->ip_summed == CHECKSUM_PARTIAL) ionic_tx_calc_csum(q, skb, desc_info); @@ -1242,16 +1638,22 @@ static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb) stats->pkts++; stats->bytes += skb->len; - if (!ionic_txq_hwstamp_enabled(q)) - netdev_tx_sent_queue(q_to_ndq(q), skb->len); - ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb); + if (likely(!ionic_txq_hwstamp_enabled(q))) { + struct netdev_queue *ndq = q_to_ndq(netdev, q); + + if (unlikely(!ionic_q_has_space(q, MAX_SKB_FRAGS + 1))) + netif_tx_stop_queue(ndq); + ring_dbell = __netdev_tx_sent_queue(ndq, skb->len, + netdev_xmit_more()); + } + ionic_txq_post(q, ring_dbell); return 0; } static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) { - struct ionic_tx_stats *stats = q_to_tx_stats(q); + int nr_frags = skb_shinfo(skb)->nr_frags; bool too_many_frags = false; skb_frag_t *frag; int desc_bufs; @@ -1267,17 +1669,20 @@ static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) /* Each desc is mss long max, so a descriptor for each gso_seg */ if (skb_is_gso(skb)) { ndescs = skb_shinfo(skb)->gso_segs; + if (!nr_frags) + return ndescs; } else { ndescs = 1; - if (skb_shinfo(skb)->nr_frags > q->max_sg_elems) { + if (!nr_frags) + return ndescs; + + if (unlikely(nr_frags > q->max_sg_elems)) { too_many_frags = true; goto linearize; } - } - /* If non-TSO, or no frags to check, we're done */ - if (!skb_is_gso(skb) || !skb_shinfo(skb)->nr_frags) return ndescs; + } /* We need to scan the skb to be sure that none of the MTU sized * packets in the TSO will require more sgs per descriptor than we @@ -1328,36 +1733,17 @@ linearize: err = skb_linearize(skb); if (err) return err; - stats->linearize++; + q_to_tx_stats(q)->linearize++; } return ndescs; } -static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs) -{ - int stopped = 0; - - if (unlikely(!ionic_q_has_space(q, ndescs))) { - netif_stop_subqueue(q->lif->netdev, q->index); - stopped = 1; - - /* Might race with ionic_tx_clean, check again */ - smp_rmb(); - if (ionic_q_has_space(q, ndescs)) { - netif_wake_subqueue(q->lif->netdev, q->index); - stopped = 0; - } - } - - return stopped; -} - static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb, struct net_device *netdev) { struct ionic_lif *lif = netdev_priv(netdev); - struct ionic_queue *q = &lif->hwstamp_txq->q; + struct ionic_queue *q; int err, ndescs; /* Does not stop/start txq, because we post to a separate tx queue @@ -1365,6 +1751,7 @@ static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb, * the timestamping queue, it is dropped. */ + q = &lif->hwstamp_txq->q; ndescs = ionic_tx_descs_needed(q, skb); if (unlikely(ndescs < 0)) goto err_out_drop; @@ -1374,9 +1761,9 @@ static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb, skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP; if (skb_is_gso(skb)) - err = ionic_tx_tso(q, skb); + err = ionic_tx_tso(netdev, q, skb); else - err = ionic_tx(q, skb); + err = ionic_tx(netdev, q, skb); if (err) goto err_out_drop; @@ -1414,23 +1801,19 @@ netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev) if (ndescs < 0) goto err_out_drop; - if (unlikely(ionic_maybe_stop_tx(q, ndescs))) + if (!netif_txq_maybe_stop(q_to_ndq(netdev, q), + ionic_q_space_avail(q), + ndescs, ndescs)) return NETDEV_TX_BUSY; if (skb_is_gso(skb)) - err = ionic_tx_tso(q, skb); + err = ionic_tx_tso(netdev, q, skb); else - err = ionic_tx(q, skb); + err = ionic_tx(netdev, q, skb); if (err) goto err_out_drop; - /* Stop the queue if there aren't descriptors for the next packet. - * Since our SG lists per descriptor take care of most of the possible - * fragmentation, we don't need to have many descriptors available. - */ - ionic_maybe_stop_tx(q, 4); - return NETDEV_TX_OK; err_out_drop: |