From e516c3fc6c182736aec5418a73f15199640491e2 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 7 Feb 2024 14:14:29 +0000 Subject: block: optimise in irq bio put caching When enlisting a bio into ->free_list_irq we protect the list by disabling irqs. It's likely they're already disabled and performance of local_irq_{save,restore}() is decent, but it's not zero cost. Let's only use the irq cache when when we're serving a hard irq, which allows to remove local_irq_{save,restore}(), and fall back to bio_free() in all left cases. Profiles indicate that the bio_put() cost is reduced by ~3.5 times (1.76% -> 0.49%), and total throughput of a CPU bound benchmark improve by around 1% (t/io_uring with high QD and several drives). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/36d207540b7046c653cc16e5ff08fe7234b19f81.1707314970.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- block/bio.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'block/bio.c') diff --git a/block/bio.c b/block/bio.c index 8da941974f88..00847ff1415c 100644 --- a/block/bio.c +++ b/block/bio.c @@ -762,30 +762,31 @@ static inline void bio_put_percpu_cache(struct bio *bio) struct bio_alloc_cache *cache; cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu()); - if (READ_ONCE(cache->nr_irq) + cache->nr > ALLOC_CACHE_MAX) { - put_cpu(); - bio_free(bio); - return; - } - - bio_uninit(bio); + if (READ_ONCE(cache->nr_irq) + cache->nr > ALLOC_CACHE_MAX) + goto out_free; if (in_task()) { + bio_uninit(bio); bio->bi_next = cache->free_list; /* Not necessary but helps not to iopoll already freed bios */ bio->bi_bdev = NULL; cache->free_list = bio; cache->nr++; - } else { - unsigned long flags; + } else if (in_hardirq()) { + lockdep_assert_irqs_disabled(); - local_irq_save(flags); + bio_uninit(bio); bio->bi_next = cache->free_list_irq; cache->free_list_irq = bio; cache->nr_irq++; - local_irq_restore(flags); + } else { + goto out_free; } put_cpu(); + return; +out_free: + put_cpu(); + bio_free(bio); } /** -- cgit v1.2.3