diff options
| author | Eric Dumazet <edumazet@google.com> | 2023-03-28 23:50:21 +0000 |
|---|---|---|
| committer | Paolo Abeni <pabeni@redhat.com> | 2023-03-30 13:40:00 +0200 |
| commit | 8b43fd3d1d7d88293eb15e92090826e6b7cc13e4 (patch) | |
| tree | 955e0aa2919e1b114b1727e7fddd538c73b3c205 /net/core | |
| parent | 821eba962d95806beb0440742c4062a9da8a386b (diff) | |
| download | linux-stable-8b43fd3d1d7d88293eb15e92090826e6b7cc13e4.tar.gz linux-stable-8b43fd3d1d7d88293eb15e92090826e6b7cc13e4.tar.bz2 linux-stable-8b43fd3d1d7d88293eb15e92090826e6b7cc13e4.zip | |
net: optimize ____napi_schedule() to avoid extra NET_RX_SOFTIRQ
____napi_schedule() adds a napi into current cpu softnet_data poll_list,
then raises NET_RX_SOFTIRQ to make sure net_rx_action() will process it.
Idea of this patch is to not raise NET_RX_SOFTIRQ when being called indirectly
from net_rx_action(), because we can process poll_list from this point,
without going to full softirq loop.
This needs a change in net_rx_action() to make sure we restart
its main loop if sd->poll_list was updated without NET_RX_SOFTIRQ
being raised.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Jason Xing <kernelxing@tencent.com>
Reviewed-by: Jason Xing <kerneljasonxing@gmail.com>
Tested-by: Jason Xing <kerneljasonxing@gmail.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Diffstat (limited to 'net/core')
| -rw-r--r-- | net/core/dev.c | 22 |
1 files changed, 18 insertions, 4 deletions
diff --git a/net/core/dev.c b/net/core/dev.c index f34ce93f2f02..0c4b21291348 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4360,7 +4360,11 @@ static inline void ____napi_schedule(struct softnet_data *sd, } list_add_tail(&napi->poll_list, &sd->poll_list); - __raise_softirq_irqoff(NET_RX_SOFTIRQ); + /* If not called from net_rx_action() + * we have to raise NET_RX_SOFTIRQ. + */ + if (!sd->in_net_rx_action) + __raise_softirq_irqoff(NET_RX_SOFTIRQ); } #ifdef CONFIG_RPS @@ -6648,6 +6652,7 @@ static __latent_entropy void net_rx_action(struct softirq_action *h) LIST_HEAD(list); LIST_HEAD(repoll); +start: sd->in_net_rx_action = true; local_irq_disable(); list_splice_init(&sd->poll_list, &list); @@ -6659,9 +6664,18 @@ static __latent_entropy void net_rx_action(struct softirq_action *h) skb_defer_free_flush(sd); if (list_empty(&list)) { - sd->in_net_rx_action = false; - if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) - goto end; + if (list_empty(&repoll)) { + sd->in_net_rx_action = false; + barrier(); + /* We need to check if ____napi_schedule() + * had refilled poll_list while + * sd->in_net_rx_action was true. + */ + if (!list_empty(&sd->poll_list)) + goto start; + if (!sd_has_rps_ipi_waiting(sd)) + goto end; + } break; } |