From 09c2c90705bb64986d499c4a4f3fd659761b637c Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 1 Aug 2023 20:52:51 +0000 Subject: net: allow alloc_skb_with_frags() to allocate bigger packets Refactor alloc_skb_with_frags() to allow bigger packets allocations. Instead of assuming that only order-0 allocations will be attempted, use the caller supplied max order. v2: try harder to use high-order pages, per Willem feedback. Link: https://lore.kernel.org/netdev/CANn89iJQfmc_KeUr3TeXvsLQwo3ZymyoCr7Y6AnHrkWSuz0yAg@mail.gmail.com/ Signed-off-by: Eric Dumazet Cc: Tahsin Erdogan Reviewed-by: Willem de Bruijn Link: https://lore.kernel.org/r/20230801205254.400094-2-edumazet@google.com Signed-off-by: Jakub Kicinski --- net/core/skbuff.c | 56 +++++++++++++++++++++++++------------------------------ 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index a298992060e6..c6f98245582c 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -6204,7 +6204,7 @@ EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); * * @header_len: size of linear part * @data_len: needed length in frags - * @max_page_order: max page order desired. + * @order: max page order desired. * @errcode: pointer to error code if any * @gfp_mask: allocation mask * @@ -6212,21 +6212,17 @@ EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); */ struct sk_buff *alloc_skb_with_frags(unsigned long header_len, unsigned long data_len, - int max_page_order, + int order, int *errcode, gfp_t gfp_mask) { - int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT; unsigned long chunk; struct sk_buff *skb; struct page *page; - int i; + int nr_frags = 0; *errcode = -EMSGSIZE; - /* Note this test could be relaxed, if we succeed to allocate - * high order pages... - */ - if (npages > MAX_SKB_FRAGS) + if (unlikely(data_len > MAX_SKB_FRAGS * (PAGE_SIZE << order))) return NULL; *errcode = -ENOBUFS; @@ -6234,34 +6230,32 @@ struct sk_buff *alloc_skb_with_frags(unsigned long header_len, if (!skb) return NULL; - skb->truesize += npages << PAGE_SHIFT; - - for (i = 0; npages > 0; i++) { - int order = max_page_order; - - while (order) { - if (npages >= 1 << order) { - page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | - __GFP_COMP | - __GFP_NOWARN, - order); - if (page) - goto fill_page; - /* Do not retry other high order allocations */ - order = 1; - max_page_order = 0; - } + while (data_len) { + if (nr_frags == MAX_SKB_FRAGS - 1) + goto failure; + while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order)) order--; + + if (order) { + page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | + __GFP_COMP | + __GFP_NOWARN, + order); + if (!page) { + order--; + continue; + } + } else { + page = alloc_page(gfp_mask); + if (!page) + goto failure; } - page = alloc_page(gfp_mask); - if (!page) - goto failure; -fill_page: chunk = min_t(unsigned long, data_len, PAGE_SIZE << order); - skb_fill_page_desc(skb, i, page, 0, chunk); + skb_fill_page_desc(skb, nr_frags, page, 0, chunk); + nr_frags++; + skb->truesize += (PAGE_SIZE << order); data_len -= chunk; - npages -= 1 << order; } return skb; -- cgit v1.2.3 From ce7c7fef147311d0675aa8351ff72e0be1d5b3eb Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 1 Aug 2023 20:52:52 +0000 Subject: net: tun: change tun_alloc_skb() to allow bigger paged allocations tun_alloc_skb() is currently calling sock_alloc_send_pskb() forcing order-0 page allocations. Switch to PAGE_ALLOC_COSTLY_ORDER, to increase max allocation size by 8x. Also add logic to increase the linear part if needed. Signed-off-by: Eric Dumazet Cc: Tahsin Erdogan Reviewed-by: Willem de Bruijn Link: https://lore.kernel.org/r/20230801205254.400094-3-edumazet@google.com Signed-off-by: Jakub Kicinski --- drivers/net/tun.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/tun.c b/drivers/net/tun.c index d75456adc62a..8a48431e8c5b 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1526,8 +1526,10 @@ static struct sk_buff *tun_alloc_skb(struct tun_file *tfile, if (prepad + len < PAGE_SIZE || !linear) linear = len; + if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) + linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER); skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, - &err, 0); + &err, PAGE_ALLOC_COSTLY_ORDER); if (!skb) return ERR_PTR(err); -- cgit v1.2.3 From ae6db08f8b5606ccd95ef2aadd741905d3c13bc0 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 1 Aug 2023 20:52:53 +0000 Subject: net/packet: change packet_alloc_skb() to allow bigger paged allocations packet_alloc_skb() is currently calling sock_alloc_send_pskb() forcing order-0 page allocations. Switch to PAGE_ALLOC_COSTLY_ORDER, to increase max size by 8x. Also add logic to increase the linear part if needed. Signed-off-by: Eric Dumazet Cc: Tahsin Erdogan Reviewed-by: Willem de Bruijn Link: https://lore.kernel.org/r/20230801205254.400094-4-edumazet@google.com Signed-off-by: Jakub Kicinski --- net/packet/af_packet.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 8e3ddec4c3d5..3b77d255d22d 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2927,8 +2927,10 @@ static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad, if (prepad + len < PAGE_SIZE || !linear) linear = len; + if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) + linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER); skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, - err, 0); + err, PAGE_ALLOC_COSTLY_ORDER); if (!skb) return NULL; -- cgit v1.2.3 From 37dfe5b8ddeb7c0bb97e3643dcfd8f9f1421ad25 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 1 Aug 2023 20:52:54 +0000 Subject: net: tap: change tap_alloc_skb() to allow bigger paged allocations tap_alloc_skb() is currently calling sock_alloc_send_pskb() forcing order-0 page allocations. Switch to PAGE_ALLOC_COSTLY_ORDER, to increase max size by 8x. Also add logic to increase the linear part if needed. Signed-off-by: Eric Dumazet Cc: Tahsin Erdogan Reviewed-by: Willem de Bruijn Link: https://lore.kernel.org/r/20230801205254.400094-5-edumazet@google.com Signed-off-by: Jakub Kicinski --- drivers/net/tap.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/tap.c b/drivers/net/tap.c index 9137fb8c1c42..01574b9d410f 100644 --- a/drivers/net/tap.c +++ b/drivers/net/tap.c @@ -614,8 +614,10 @@ static inline struct sk_buff *tap_alloc_skb(struct sock *sk, size_t prepad, if (prepad + len < PAGE_SIZE || !linear) linear = len; + if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) + linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER); skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, - err, 0); + err, PAGE_ALLOC_COSTLY_ORDER); if (!skb) return NULL; -- cgit v1.2.3