summaryrefslogtreecommitdiffstats
path: root/net/core
diff options
context:
space:
mode:
authorEric Dumazet <edumazet@google.com>2014-10-03 15:31:07 -0700
committerDavid S. Miller <davem@davemloft.net>2014-10-03 15:36:11 -0700
commit55a93b3ea780908b7d1b3a8cf1976223a9268d78 (patch)
tree2c33ad6aaf7a604185bff5659d64895f5cf9a348 /net/core
parent6a05880a8b22c6ba2ffdabbceb4635d28abe9072 (diff)
downloadlinux-55a93b3ea780908b7d1b3a8cf1976223a9268d78.tar.gz
linux-55a93b3ea780908b7d1b3a8cf1976223a9268d78.tar.bz2
linux-55a93b3ea780908b7d1b3a8cf1976223a9268d78.zip
qdisc: validate skb without holding lock
Validation of skb can be pretty expensive : GSO segmentation and/or checksum computations. We can do this without holding qdisc lock, so that other cpus can queue additional packets. Trick is that requeued packets were already validated, so we carry a boolean so that sch_direct_xmit() can validate a fresh skb list, or directly use an old one. Tested on 40Gb NIC (8 TX queues) and 200 concurrent flows, 48 threads host. Turning TSO on or off had no effect on throughput, only few more cpu cycles. Lock contention on qdisc lock disappeared. Same if disabling TX checksum offload. Signed-off-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net/core')
-rw-r--r--net/core/dev.c29
1 files changed, 26 insertions, 3 deletions
diff --git a/net/core/dev.c b/net/core/dev.c
index e55c546717d4..1a90530f83ff 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2655,7 +2655,7 @@ struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, netdev_features_t featur
return skb;
}
-struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
+static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
{
netdev_features_t features;
@@ -2720,6 +2720,30 @@ out_null:
return NULL;
}
+struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
+{
+ struct sk_buff *next, *head = NULL, *tail;
+
+ while (skb) {
+ next = skb->next;
+ skb->next = NULL;
+ skb = validate_xmit_skb(skb, dev);
+ if (skb) {
+ struct sk_buff *end = skb;
+
+ while (end->next)
+ end = end->next;
+ if (!head)
+ head = skb;
+ else
+ tail->next = skb;
+ tail = end;
+ }
+ skb = next;
+ }
+ return head;
+}
+
static void qdisc_pkt_len_init(struct sk_buff *skb)
{
const struct skb_shared_info *shinfo = skb_shinfo(skb);
@@ -2786,8 +2810,7 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
qdisc_bstats_update(q, skb);
- skb = validate_xmit_skb(skb, dev);
- if (skb && sch_direct_xmit(skb, q, dev, txq, root_lock)) {
+ if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
if (unlikely(contended)) {
spin_unlock(&q->busylock);
contended = false;