diff options
author | David S. Miller <davem@davemloft.net> | 2018-04-01 14:08:21 -0400 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2018-04-01 14:08:21 -0400 |
commit | 16c3c91346961fdd733ad232311772795599be7f (patch) | |
tree | 253468761ee8e0f7a7cd97f21c24ef3c841529e7 | |
parent | c07255020551cadae8bf903f2c5e1fcbad731bac (diff) | |
parent | 1f4c6eb24029689a40dceae561e31ff6926d7f0d (diff) | |
download | linux-stable-16c3c91346961fdd733ad232311772795599be7f.tar.gz linux-stable-16c3c91346961fdd733ad232311772795599be7f.tar.bz2 linux-stable-16c3c91346961fdd733ad232311772795599be7f.zip |
Merge branch 'inet-factorize-sk_wmem_alloc-updates'
Eric Dumazet says:
====================
inet: factorize sk_wmem_alloc updates
While testing my inet defrag changes, I found that senders
could spend ~20% of cpu cycles in skb_set_owner_w() updating
sk->sk_wmem_alloc for every fragment they cook, competing
with TX completion of prior skbs possibly happening on another cpus.
One solution to this problem is to use alloc_skb() instead
of sock_wmalloc() and manually perform a single sk_wmem_alloc change.
This greatly increases speed for applications sending big UDP datagrams.
====================
Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r-- | net/ipv4/ip_output.c | 17 | ||||
-rw-r--r-- | net/ipv6/ip6_output.c | 17 |
2 files changed, 24 insertions, 10 deletions
diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 66340ab750e6..94cacae76aca 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -876,6 +876,7 @@ static int __ip_append_data(struct sock *sk, unsigned int maxfraglen, fragheaderlen, maxnonfragsize; int csummode = CHECKSUM_NONE; struct rtable *rt = (struct rtable *)cork->dst; + unsigned int wmem_alloc_delta = 0; u32 tskey = 0; skb = skb_peek_tail(queue); @@ -971,11 +972,10 @@ alloc_new_skb: (flags & MSG_DONTWAIT), &err); } else { skb = NULL; - if (refcount_read(&sk->sk_wmem_alloc) <= + if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <= 2 * sk->sk_sndbuf) - skb = sock_wmalloc(sk, - alloclen + hh_len + 15, 1, - sk->sk_allocation); + skb = alloc_skb(alloclen + hh_len + 15, + sk->sk_allocation); if (unlikely(!skb)) err = -ENOBUFS; } @@ -1033,6 +1033,11 @@ alloc_new_skb: /* * Put the packet on the pending queue. */ + if (!skb->destructor) { + skb->destructor = sock_wfree; + skb->sk = sk; + wmem_alloc_delta += skb->truesize; + } __skb_queue_tail(queue, skb); continue; } @@ -1079,12 +1084,13 @@ alloc_new_skb: skb->len += copy; skb->data_len += copy; skb->truesize += copy; - refcount_add(copy, &sk->sk_wmem_alloc); + wmem_alloc_delta += copy; } offset += copy; length -= copy; } + refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); return 0; error_efault: @@ -1092,6 +1098,7 @@ error_efault: error: cork->length -= length; IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS); + refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); return err; } diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index 2c7f09c3c39e..323d7a354ffb 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1259,6 +1259,7 @@ static int __ip6_append_data(struct sock *sk, struct ipv6_txoptions *opt = v6_cork->opt; int csummode = CHECKSUM_NONE; unsigned int maxnonfragsize, headersize; + unsigned int wmem_alloc_delta = 0; skb = skb_peek_tail(queue); if (!skb) { @@ -1411,11 +1412,10 @@ alloc_new_skb: (flags & MSG_DONTWAIT), &err); } else { skb = NULL; - if (refcount_read(&sk->sk_wmem_alloc) <= + if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <= 2 * sk->sk_sndbuf) - skb = sock_wmalloc(sk, - alloclen + hh_len, 1, - sk->sk_allocation); + skb = alloc_skb(alloclen + hh_len, + sk->sk_allocation); if (unlikely(!skb)) err = -ENOBUFS; } @@ -1474,6 +1474,11 @@ alloc_new_skb: /* * Put the packet on the pending queue */ + if (!skb->destructor) { + skb->destructor = sock_wfree; + skb->sk = sk; + wmem_alloc_delta += skb->truesize; + } __skb_queue_tail(queue, skb); continue; } @@ -1520,12 +1525,13 @@ alloc_new_skb: skb->len += copy; skb->data_len += copy; skb->truesize += copy; - refcount_add(copy, &sk->sk_wmem_alloc); + wmem_alloc_delta += copy; } offset += copy; length -= copy; } + refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); return 0; error_efault: @@ -1533,6 +1539,7 @@ error_efault: error: cork->length -= length; IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); + refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); return err; } |