summaryrefslogtreecommitdiffstats
path: root/mm/mmu_gather.c
diff options
context:
space:
mode:
authorWill Deacon <will@kernel.org>2021-01-27 23:53:44 +0000
committerPeter Zijlstra <peterz@infradead.org>2021-01-29 20:02:29 +0100
commitd8b450530b90f8845ab962af18b8a10ed77fc889 (patch)
tree5399ddb38a5f0a743c07866a32bec5ce46175b61 /mm/mmu_gather.c
parentae8eba8b5d723a4ca543024b6e51f4d0f4fb6b6b (diff)
downloadlinux-d8b450530b90f8845ab962af18b8a10ed77fc889.tar.gz
linux-d8b450530b90f8845ab962af18b8a10ed77fc889.tar.bz2
linux-d8b450530b90f8845ab962af18b8a10ed77fc889.zip
tlb: mmu_gather: Introduce tlb_gather_mmu_fullmm()
Passing the range '0, -1' to tlb_gather_mmu() sets the 'fullmm' flag, which indicates that the mm_struct being operated on is going away. In this case, some architectures (such as arm64) can elide TLB invalidation by ensuring that the TLB tag (ASID) associated with this mm is not immediately reclaimed. Although this behaviour is documented in asm-generic/tlb.h, it's subtle and easily missed. Introduce tlb_gather_mmu_fullmm() to make it clearer that this is for the entire mm and WARN() if tlb_gather_mmu() is called with the 'fullmm' address range. Signed-off-by: Will Deacon <will@kernel.org> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Reviewed-by: Yu Zhao <yuzhao@google.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Linus Torvalds <torvalds@linux-foundation.org> Link: https://lkml.kernel.org/r/20210127235347.1402-4-will@kernel.org
Diffstat (limited to 'mm/mmu_gather.c')
-rw-r--r--mm/mmu_gather.c16
1 files changed, 14 insertions, 2 deletions
diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c
index b0be5a7aa08f..5f5e45d9eb50 100644
--- a/mm/mmu_gather.c
+++ b/mm/mmu_gather.c
@@ -261,8 +261,8 @@ void tlb_flush_mmu(struct mmu_gather *tlb)
* respectively when @mm is without users and we're going to destroy
* the full address space (exit/execve).
*/
-void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
- unsigned long start, unsigned long end)
+static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
+ unsigned long start, unsigned long end)
{
tlb->mm = mm;
@@ -287,6 +287,18 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
inc_tlb_flush_pending(tlb->mm);
}
+void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
+ unsigned long start, unsigned long end)
+{
+ WARN_ON(!(start | (end + 1))); /* Use _fullmm() instead */
+ __tlb_gather_mmu(tlb, mm, start, end);
+}
+
+void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm)
+{
+ __tlb_gather_mmu(tlb, mm, 0, -1);
+}
+
/**
* tlb_finish_mmu - finish an mmu_gather structure
* @tlb: the mmu_gather structure to finish