diff options
author | Mel Gorman <mgorman@techsingularity.net> | 2022-03-22 14:43:36 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2022-03-22 15:57:06 -0700 |
commit | fd56eef258a17bbc8eda2ca773fa538f354c5f49 (patch) | |
tree | fe9adbd5e7504c115c100dccbba9a991d3cc9574 /mm/page_alloc.c | |
parent | 35b6d770e6334aa470080570f0f81c8b74a07afd (diff) | |
download | linux-fd56eef258a17bbc8eda2ca773fa538f354c5f49.tar.gz linux-fd56eef258a17bbc8eda2ca773fa538f354c5f49.tar.bz2 linux-fd56eef258a17bbc8eda2ca773fa538f354c5f49.zip |
mm/page_alloc: simplify how many pages are selected per pcp list during bulk free
free_pcppages_bulk() selects pages to free by round-robining between
lists. Originally this was to evenly shrink pages by migratetype but
uneven freeing is inevitable due to high pages. Simplify list selection
by starting with a list that definitely has pages on it in
free_unref_page_commit() and for drain, it does not matter where
draining starts as all pages are removed.
Link: https://lkml.kernel.org/r/20220217002227.5739-4-mgorman@techsingularity.net
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Tested-by: Aaron Lu <aaron.lu@intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Michal Hocko <mhocko@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'mm/page_alloc.c')
-rw-r--r-- | mm/page_alloc.c | 34 |
1 files changed, 11 insertions, 23 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 63edba38ebc2..21f68377d40f 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1444,13 +1444,11 @@ static inline void prefetch_buddy(struct page *page, unsigned int order) * count is the number of pages to free. */ static void free_pcppages_bulk(struct zone *zone, int count, - struct per_cpu_pages *pcp) + struct per_cpu_pages *pcp, + int pindex) { - int pindex = 0; int min_pindex = 0; int max_pindex = NR_PCP_LISTS - 1; - int batch_free = 0; - int nr_freed = 0; unsigned int order; int prefetch_nr = READ_ONCE(pcp->batch); bool isolated_pageblocks; @@ -1464,16 +1462,10 @@ static void free_pcppages_bulk(struct zone *zone, int count, count = min(pcp->count, count); while (count > 0) { struct list_head *list; + int nr_pages; - /* - * Remove pages from lists in a round-robin fashion. A - * batch_free count is maintained that is incremented when an - * empty list is encountered. This is so more pages are freed - * off fuller lists instead of spinning excessively around empty - * lists - */ + /* Remove pages from lists in a round-robin fashion. */ do { - batch_free++; if (++pindex > max_pindex) pindex = min_pindex; list = &pcp->lists[pindex]; @@ -1486,18 +1478,15 @@ static void free_pcppages_bulk(struct zone *zone, int count, min_pindex++; } while (1); - /* This is the only non-empty list. Free them all. */ - if (batch_free >= max_pindex - min_pindex) - batch_free = count; - order = pindex_to_order(pindex); + nr_pages = 1 << order; BUILD_BUG_ON(MAX_ORDER >= (1<<NR_PCP_ORDER_WIDTH)); do { page = list_last_entry(list, struct page, lru); /* must delete to avoid corrupting pcp list */ list_del(&page->lru); - nr_freed += 1 << order; - count -= 1 << order; + count -= nr_pages; + pcp->count -= nr_pages; if (bulkfree_pcp_prepare(page)) continue; @@ -1521,9 +1510,8 @@ static void free_pcppages_bulk(struct zone *zone, int count, prefetch_buddy(page, order); prefetch_nr--; } - } while (count > 0 && --batch_free && !list_empty(list)); + } while (count > 0 && !list_empty(list)); } - pcp->count -= nr_freed; /* * local_lock_irq held so equivalent to spin_lock_irqsave for @@ -3077,7 +3065,7 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) batch = READ_ONCE(pcp->batch); to_drain = min(pcp->count, batch); if (to_drain > 0) - free_pcppages_bulk(zone, to_drain, pcp); + free_pcppages_bulk(zone, to_drain, pcp, 0); local_unlock_irqrestore(&pagesets.lock, flags); } #endif @@ -3098,7 +3086,7 @@ static void drain_pages_zone(unsigned int cpu, struct zone *zone) pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); if (pcp->count) - free_pcppages_bulk(zone, pcp->count, pcp); + free_pcppages_bulk(zone, pcp->count, pcp, 0); local_unlock_irqrestore(&pagesets.lock, flags); } @@ -3379,7 +3367,7 @@ static void free_unref_page_commit(struct page *page, int migratetype, if (pcp->count >= high) { int batch = READ_ONCE(pcp->batch); - free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch), pcp); + free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch), pcp, pindex); } } |