diff options
author | Peter Zijlstra <a.p.zijlstra@chello.nl> | 2011-05-24 17:12:00 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2011-05-25 08:39:16 -0700 |
commit | 267239116987d64850ad2037d8e0f3071dc3b5ce (patch) | |
tree | 142595897f7fc7bb673b791891dcc2fab31f6e91 /include/asm-generic/tlb.h | |
parent | 1c395176962176660bb108f90e97e1686cfe0d85 (diff) | |
download | linux-267239116987d64850ad2037d8e0f3071dc3b5ce.tar.gz linux-267239116987d64850ad2037d8e0f3071dc3b5ce.tar.bz2 linux-267239116987d64850ad2037d8e0f3071dc3b5ce.zip |
mm, powerpc: move the RCU page-table freeing into generic code
In case other architectures require RCU freed page-tables to implement
gup_fast() and software filled hashes and similar things, provide the
means to do so by moving the logic into generic code.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Requested-by: David Miller <davem@davemloft.net>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Russell King <rmk@arm.linux.org.uk>
Cc: Paul Mundt <lethal@linux-sh.org>
Cc: Jeff Dike <jdike@addtoit.com>
Cc: Richard Weinberger <richard@nod.at>
Cc: Tony Luck <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Nick Piggin <npiggin@kernel.dk>
Cc: Namhyung Kim <namhyung@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'include/asm-generic/tlb.h')
-rw-r--r-- | include/asm-generic/tlb.h | 56 |
1 files changed, 51 insertions, 5 deletions
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 2d3547c84235..74f80f6b6cf1 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -29,6 +29,49 @@ #define tlb_fast_mode(tlb) 1 #endif +#ifdef CONFIG_HAVE_RCU_TABLE_FREE +/* + * Semi RCU freeing of the page directories. + * + * This is needed by some architectures to implement software pagetable walkers. + * + * gup_fast() and other software pagetable walkers do a lockless page-table + * walk and therefore needs some synchronization with the freeing of the page + * directories. The chosen means to accomplish that is by disabling IRQs over + * the walk. + * + * Architectures that use IPIs to flush TLBs will then automagically DTRT, + * since we unlink the page, flush TLBs, free the page. Since the disabling of + * IRQs delays the completion of the TLB flush we can never observe an already + * freed page. + * + * Architectures that do not have this (PPC) need to delay the freeing by some + * other means, this is that means. + * + * What we do is batch the freed directory pages (tables) and RCU free them. + * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling + * holds off grace periods. + * + * However, in order to batch these pages we need to allocate storage, this + * allocation is deep inside the MM code and can thus easily fail on memory + * pressure. To guarantee progress we fall back to single table freeing, see + * the implementation of tlb_remove_table_one(). + * + */ +struct mmu_table_batch { + struct rcu_head rcu; + unsigned int nr; + void *tables[0]; +}; + +#define MAX_TABLE_BATCH \ + ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *)) + +extern void tlb_table_flush(struct mmu_gather *tlb); +extern void tlb_remove_table(struct mmu_gather *tlb, void *table); + +#endif + /* * If we can't allocate a page to make a big batch of page pointers * to work on, then just handle a few from the on-stack structure. @@ -40,13 +83,13 @@ */ struct mmu_gather { struct mm_struct *mm; +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + struct mmu_table_batch *batch; +#endif unsigned int nr; /* set to ~0U means fast mode */ unsigned int max; /* nr < max */ unsigned int need_flush;/* Really unmapped some ptes? */ unsigned int fullmm; /* non-zero means full mm flush */ -#ifdef HAVE_ARCH_MMU_GATHER - struct arch_mmu_gather arch; -#endif struct page **pages; struct page *local[MMU_GATHER_BUNDLE]; }; @@ -82,8 +125,8 @@ tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm) tlb->fullmm = fullmm; -#ifdef HAVE_ARCH_MMU_GATHER - tlb->arch = ARCH_MMU_GATHER_INIT; +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + tlb->batch = NULL; #endif } @@ -94,6 +137,9 @@ tlb_flush_mmu(struct mmu_gather *tlb) return; tlb->need_flush = 0; tlb_flush(tlb); +#ifdef CONFIG_HAVE_RCU_TABLE_FREE + tlb_table_flush(tlb); +#endif if (!tlb_fast_mode(tlb)) { free_pages_and_swap_cache(tlb->pages, tlb->nr); tlb->nr = 0; |