summaryrefslogtreecommitdiffstats
path: root/mm/percpu.c
diff options
context:
space:
mode:
authorYury Norov <yury.norov@gmail.com>2021-08-14 14:17:10 -0700
committerYury Norov <yury.norov@gmail.com>2022-01-15 08:47:31 -0800
commit801a57365fc836d7ec866e2069d0b21d79925c1e (patch)
tree76af2ac9496f91c85eb00a66be42235c82a1bbc6 /mm/percpu.c
parent749443de8dde3b8b420ee8b4daac4d929a6adeb9 (diff)
downloadlinux-stable-801a57365fc836d7ec866e2069d0b21d79925c1e.tar.gz
linux-stable-801a57365fc836d7ec866e2069d0b21d79925c1e.tar.bz2
linux-stable-801a57365fc836d7ec866e2069d0b21d79925c1e.zip
mm/percpu: micro-optimize pcpu_is_populated()
bitmap_next_clear_region() calls find_next_zero_bit() and find_next_bit() sequentially to find a range of clear bits. In case of pcpu_is_populated() there's a chance to return earlier if bitmap has all bits set. Signed-off-by: Yury Norov <yury.norov@gmail.com> Tested-by: Wolfram Sang <wsa+renesas@sang-engineering.com> Acked-by: Dennis Zhou <dennis@kernel.org>
Diffstat (limited to 'mm/percpu.c')
-rw-r--r--mm/percpu.c15
1 files changed, 8 insertions, 7 deletions
diff --git a/mm/percpu.c b/mm/percpu.c
index f5b2c2ea5a54..0c6f85c5514f 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1070,17 +1070,18 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits,
int *next_off)
{
- unsigned int page_start, page_end, rs, re;
+ unsigned int start, end;
- page_start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE);
- page_end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);
+ start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE);
+ end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);
- rs = page_start;
- bitmap_next_clear_region(chunk->populated, &rs, &re, page_end);
- if (rs >= page_end)
+ start = find_next_zero_bit(chunk->populated, end, start);
+ if (start >= end)
return true;
- *next_off = re * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;
+ end = find_next_bit(chunk->populated, end, start + 1);
+
+ *next_off = end * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;
return false;
}