diff options
| author | Matthew Wilcox (Oracle) <willy@infradead.org> | 2023-08-16 16:11:50 +0100 |
|---|---|---|
| committer | Andrew Morton <akpm@linux-foundation.org> | 2023-08-21 14:28:43 -0700 |
| commit | dd6fa0b61814492476463149c91110e529364e82 (patch) | |
| tree | 79f3c0ef997a8583ca3c4d2fd6faa4d446caa65f | |
| parent | 99a9e0b83ab9955e604397717b82267feb021df3 (diff) | |
| download | linux-dd6fa0b61814492476463149c91110e529364e82.tar.gz linux-dd6fa0b61814492476463149c91110e529364e82.tar.bz2 linux-dd6fa0b61814492476463149c91110e529364e82.zip | |
mm: call free_huge_page() directly
Indirect calls are expensive, thanks to Spectre. Call free_huge_page()
directly if the folio belongs to hugetlb.
Link: https://lkml.kernel.org/r/20230816151201.3655946-3-willy@infradead.org
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Yanteng Si <siyanteng@loongson.cn>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
| -rw-r--r-- | include/linux/hugetlb.h | 3 | ||||
| -rw-r--r-- | mm/page_alloc.c | 8 |
2 files changed, 7 insertions, 4 deletions
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 0a393bc02f25..5a1dfaffbd80 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -26,6 +26,8 @@ typedef struct { unsigned long pd; } hugepd_t; #define __hugepd(x) ((hugepd_t) { (x) }) #endif +void free_huge_page(struct page *page); + #ifdef CONFIG_HUGETLB_PAGE #include <linux/mempolicy.h> @@ -165,7 +167,6 @@ int get_huge_page_for_hwpoison(unsigned long pfn, int flags, bool *migratable_cleared); void folio_putback_active_hugetlb(struct folio *folio); void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason); -void free_huge_page(struct page *page); void hugetlb_fix_reserve_counts(struct inode *inode); extern struct mutex *hugetlb_fault_mutex_table; u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 986b56db96b5..744848593360 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -287,9 +287,6 @@ const char * const migratetype_names[MIGRATE_TYPES] = { static compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = { [NULL_COMPOUND_DTOR] = NULL, [COMPOUND_PAGE_DTOR] = free_compound_page, -#ifdef CONFIG_HUGETLB_PAGE - [HUGETLB_PAGE_DTOR] = free_huge_page, -#endif #ifdef CONFIG_TRANSPARENT_HUGEPAGE [TRANSHUGE_PAGE_DTOR] = free_transhuge_page, #endif @@ -612,6 +609,11 @@ void destroy_large_folio(struct folio *folio) { enum compound_dtor_id dtor = folio->_folio_dtor; + if (folio_test_hugetlb(folio)) { + free_huge_page(&folio->page); + return; + } + VM_BUG_ON_FOLIO(dtor >= NR_COMPOUND_DTORS, folio); compound_page_dtors[dtor](&folio->page); } |