diff options
Diffstat (limited to 'arch/powerpc/include/asm/book3s')
| -rw-r--r-- | arch/powerpc/include/asm/book3s/32/pgalloc.h | 1 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/32/pgtable.h | 7 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/hash-4k.h | 8 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/hash-64k.h | 7 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/hash.h | 10 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/mmu.h | 7 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/pgalloc.h | 46 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/pgtable.h | 28 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/radix-4k.h | 3 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/radix-64k.h | 4 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/radix.h | 86 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/tlbflush-radix.h | 7 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/tlbflush.h | 12 |
13 files changed, 118 insertions, 108 deletions
diff --git a/arch/powerpc/include/asm/book3s/32/pgalloc.h b/arch/powerpc/include/asm/book3s/32/pgalloc.h index 5073cc75f1c8..6a6673907e45 100644 --- a/arch/powerpc/include/asm/book3s/32/pgalloc.h +++ b/arch/powerpc/include/asm/book3s/32/pgalloc.h @@ -99,6 +99,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) static inline void pgtable_free(void *table, unsigned index_size) { if (!index_size) { + pgtable_page_dtor(virt_to_page(table)); free_page((unsigned long)table); } else { BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE); diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index c615abdce119..02f5acd7ccc4 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -235,15 +235,18 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, } -static inline void __ptep_set_access_flags(struct mm_struct *mm, +static inline void __ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, pte_t entry, - unsigned long address) + unsigned long address, + int psize) { unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC); unsigned long clr = ~pte_val(entry) & _PAGE_RO; pte_update(ptep, clr, set); + + flush_tlb_page(vma, address); } #define __HAVE_ARCH_PTE_SAME diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h index 4b5423030d4b..9a3798660cef 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-4k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h @@ -38,8 +38,12 @@ #define H_PAGE_4K_PFN 0x0 #define H_PAGE_THP_HUGE 0x0 #define H_PAGE_COMBO 0x0 -#define H_PTE_FRAG_NR 0 -#define H_PTE_FRAG_SIZE_SHIFT 0 + +/* 8 bytes per each pte entry */ +#define H_PTE_FRAG_SIZE_SHIFT (H_PTE_INDEX_SIZE + 3) +#define H_PTE_FRAG_NR (PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT) +#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3) +#define H_PMD_FRAG_NR (PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT) /* memory key bits, only 8 keys supported */ #define H_PTE_PKEY_BIT0 0 diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index cc82745355b3..c81793d47af9 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -46,6 +46,13 @@ #define H_PTE_FRAG_SIZE_SHIFT (H_PTE_INDEX_SIZE + 3 + 1) #define H_PTE_FRAG_NR (PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT) +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) +#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3 + 1) +#else +#define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3) +#endif +#define H_PMD_FRAG_NR (PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT) + #ifndef __ASSEMBLY__ #include <asm/errno.h> diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index cc8cd656ccfe..0387b155f13d 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -23,16 +23,6 @@ H_PUD_INDEX_SIZE + H_PGD_INDEX_SIZE + PAGE_SHIFT) #define H_PGTABLE_RANGE (ASM_CONST(1) << H_PGTABLE_EADDR_SIZE) -#if (defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)) && \ - defined(CONFIG_PPC_64K_PAGES) -/* - * only with hash 64k we need to use the second half of pmd page table - * to store pointer to deposited pgtable_t - */ -#define H_PMD_CACHE_INDEX (H_PMD_INDEX_SIZE + 1) -#else -#define H_PMD_CACHE_INDEX H_PMD_INDEX_SIZE -#endif /* * We store the slot details in the second half of page table. * Increase the pud level table so that hugetlb ptes can be stored diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h index 5094696eecd6..9c8c669a6b6a 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu.h +++ b/arch/powerpc/include/asm/book3s/64/mmu.h @@ -134,10 +134,11 @@ typedef struct { #ifdef CONFIG_PPC_SUBPAGE_PROT struct subpage_prot_table spt; #endif /* CONFIG_PPC_SUBPAGE_PROT */ -#ifdef CONFIG_PPC_64K_PAGES - /* for 4K PTE fragment support */ + /* + * pagetable fragment support + */ void *pte_frag; -#endif + void *pmd_frag; #ifdef CONFIG_SPAPR_TCE_IOMMU struct list_head iommu_group_mem_list; #endif diff --git a/arch/powerpc/include/asm/book3s/64/pgalloc.h b/arch/powerpc/include/asm/book3s/64/pgalloc.h index 558a159600ad..01ee40f11f3a 100644 --- a/arch/powerpc/include/asm/book3s/64/pgalloc.h +++ b/arch/powerpc/include/asm/book3s/64/pgalloc.h @@ -42,7 +42,9 @@ extern struct kmem_cache *pgtable_cache[]; }) extern pte_t *pte_fragment_alloc(struct mm_struct *, unsigned long, int); +extern pmd_t *pmd_fragment_alloc(struct mm_struct *, unsigned long); extern void pte_fragment_free(unsigned long *, int); +extern void pmd_fragment_free(unsigned long *); extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift); #ifdef CONFIG_SMP extern void __tlb_remove_table(void *_table); @@ -88,8 +90,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) * need to do this for 4k. */ #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_64K_PAGES) && \ - ((H_PGD_INDEX_SIZE == H_PUD_CACHE_INDEX) || \ - (H_PGD_INDEX_SIZE == H_PMD_CACHE_INDEX)) + (H_PGD_INDEX_SIZE == H_PUD_CACHE_INDEX) memset(pgd, 0, PGD_TABLE_SIZE); #endif return pgd; @@ -124,36 +125,35 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) } static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, - unsigned long address) + unsigned long address) { /* * By now all the pud entries should be none entries. So go * ahead and flush the page walk cache */ flush_tlb_pgtable(tlb, address); - pgtable_free_tlb(tlb, pud, PUD_CACHE_INDEX); + pgtable_free_tlb(tlb, pud, PUD_INDEX); } static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX), - pgtable_gfp_flags(mm, GFP_KERNEL)); + return pmd_fragment_alloc(mm, addr); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) { - kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd); + pmd_fragment_free((unsigned long *)pmd); } static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, - unsigned long address) + unsigned long address) { /* * By now all the pud entries should be none entries. So go * ahead and flush the page walk cache */ flush_tlb_pgtable(tlb, address); - return pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX); + return pgtable_free_tlb(tlb, pmd, PMD_INDEX); } static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, @@ -173,31 +173,6 @@ static inline pgtable_t pmd_pgtable(pmd_t pmd) return (pgtable_t)pmd_page_vaddr(pmd); } -#ifdef CONFIG_PPC_4K_PAGES -static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, - unsigned long address) -{ - return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); -} - -static inline pgtable_t pte_alloc_one(struct mm_struct *mm, - unsigned long address) -{ - struct page *page; - pte_t *pte; - - pte = (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT); - if (!pte) - return NULL; - page = virt_to_page(pte); - if (!pgtable_page_ctor(page)) { - __free_page(page); - return NULL; - } - return pte; -} -#else /* if CONFIG_PPC_64K_PAGES */ - static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { @@ -209,7 +184,6 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, { return (pgtable_t)pte_fragment_alloc(mm, address, 0); } -#endif static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) { @@ -229,7 +203,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, * ahead and flush the page walk cache */ flush_tlb_pgtable(tlb, address); - pgtable_free_tlb(tlb, table, 0); + pgtable_free_tlb(tlb, table, PTE_INDEX); } #define check_pgt_cache() do { } while (0) diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 47b5ffc8715d..42fe7c2ff2df 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -212,13 +212,13 @@ extern unsigned long __pte_index_size; extern unsigned long __pmd_index_size; extern unsigned long __pud_index_size; extern unsigned long __pgd_index_size; -extern unsigned long __pmd_cache_index; extern unsigned long __pud_cache_index; #define PTE_INDEX_SIZE __pte_index_size #define PMD_INDEX_SIZE __pmd_index_size #define PUD_INDEX_SIZE __pud_index_size #define PGD_INDEX_SIZE __pgd_index_size -#define PMD_CACHE_INDEX __pmd_cache_index +/* pmd table use page table fragments */ +#define PMD_CACHE_INDEX 0 #define PUD_CACHE_INDEX __pud_cache_index /* * Because of use of pte fragments and THP, size of page table @@ -246,6 +246,12 @@ extern unsigned long __pte_frag_size_shift; #define PTE_FRAG_SIZE_SHIFT __pte_frag_size_shift #define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT) +extern unsigned long __pmd_frag_nr; +#define PMD_FRAG_NR __pmd_frag_nr +extern unsigned long __pmd_frag_size_shift; +#define PMD_FRAG_SIZE_SHIFT __pmd_frag_size_shift +#define PMD_FRAG_SIZE (1UL << PMD_FRAG_SIZE_SHIFT) + #define PTRS_PER_PTE (1 << PTE_INDEX_SIZE) #define PTRS_PER_PMD (1 << PMD_INDEX_SIZE) #define PTRS_PER_PUD (1 << PUD_INDEX_SIZE) @@ -273,6 +279,16 @@ extern unsigned long __pte_frag_size_shift; /* Bits to mask out from a PGD to get to the PUD page */ #define PGD_MASKED_BITS 0xc0000000000000ffUL +/* + * Used as an indicator for rcu callback functions + */ +enum pgtable_index { + PTE_INDEX = 0, + PMD_INDEX, + PUD_INDEX, + PGD_INDEX, +}; + extern unsigned long __vmalloc_start; extern unsigned long __vmalloc_end; #define VMALLOC_START __vmalloc_start @@ -751,12 +767,14 @@ static inline bool check_pte_access(unsigned long access, unsigned long ptev) * Generic functions with hash/radix callbacks */ -static inline void __ptep_set_access_flags(struct mm_struct *mm, +static inline void __ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, pte_t entry, - unsigned long address) + unsigned long address, + int psize) { if (radix_enabled()) - return radix__ptep_set_access_flags(mm, ptep, entry, address); + return radix__ptep_set_access_flags(vma, ptep, entry, + address, psize); return hash__ptep_set_access_flags(ptep, entry); } diff --git a/arch/powerpc/include/asm/book3s/64/radix-4k.h b/arch/powerpc/include/asm/book3s/64/radix-4k.h index ca366ec86310..863c3e8286fb 100644 --- a/arch/powerpc/include/asm/book3s/64/radix-4k.h +++ b/arch/powerpc/include/asm/book3s/64/radix-4k.h @@ -15,4 +15,7 @@ #define RADIX_PTE_FRAG_SIZE_SHIFT (RADIX_PTE_INDEX_SIZE + 3) #define RADIX_PTE_FRAG_NR (PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT) +#define RADIX_PMD_FRAG_SIZE_SHIFT (RADIX_PMD_INDEX_SIZE + 3) +#define RADIX_PMD_FRAG_NR (PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT) + #endif /* _ASM_POWERPC_PGTABLE_RADIX_4K_H */ diff --git a/arch/powerpc/include/asm/book3s/64/radix-64k.h b/arch/powerpc/include/asm/book3s/64/radix-64k.h index 830082496876..ccb78ca9d0c5 100644 --- a/arch/powerpc/include/asm/book3s/64/radix-64k.h +++ b/arch/powerpc/include/asm/book3s/64/radix-64k.h @@ -16,4 +16,8 @@ */ #define RADIX_PTE_FRAG_SIZE_SHIFT (RADIX_PTE_INDEX_SIZE + 3) #define RADIX_PTE_FRAG_NR (PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT) + +#define RADIX_PMD_FRAG_SIZE_SHIFT (RADIX_PMD_INDEX_SIZE + 3) +#define RADIX_PMD_FRAG_NR (PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT) + #endif /* _ASM_POWERPC_PGTABLE_RADIX_64K_H */ diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h index 705193e7192f..ef9f96742ce1 100644 --- a/arch/powerpc/include/asm/book3s/64/radix.h +++ b/arch/powerpc/include/asm/book3s/64/radix.h @@ -124,23 +124,28 @@ extern void radix__mark_rodata_ro(void); extern void radix__mark_initmem_nx(void); #endif +extern void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, + pte_t entry, unsigned long address, + int psize); + static inline unsigned long __radix_pte_update(pte_t *ptep, unsigned long clr, unsigned long set) { - pte_t pte; - unsigned long old_pte, new_pte; - - do { - pte = READ_ONCE(*ptep); - old_pte = pte_val(pte); - new_pte = (old_pte | set) & ~clr; - - } while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte))); - - return old_pte; + __be64 old_be, tmp_be; + + __asm__ __volatile__( + "1: ldarx %0,0,%3 # pte_update\n" + " andc %1,%0,%5 \n" + " or %1,%1,%4 \n" + " stdcx. %1,0,%3 \n" + " bne- 1b" + : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) + : "r" (ptep), "r" (cpu_to_be64(set)), "r" (cpu_to_be64(clr)) + : "cc" ); + + return be64_to_cpu(old_be); } - static inline unsigned long radix__pte_update(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long clr, @@ -176,48 +181,14 @@ static inline pte_t radix__ptep_get_and_clear_full(struct mm_struct *mm, unsigned long old_pte; if (full) { - /* - * If we are trying to clear the pte, we can skip - * the DD1 pte update sequence and batch the tlb flush. The - * tlb flush batching is done by mmu gather code. We - * still keep the cmp_xchg update to make sure we get - * correct R/C bit which might be updated via Nest MMU. - */ - old_pte = __radix_pte_update(ptep, ~0ul, 0); + old_pte = pte_val(*ptep); + *ptep = __pte(0); } else old_pte = radix__pte_update(mm, addr, ptep, ~0ul, 0, 0); return __pte(old_pte); } -/* - * Set the dirty and/or accessed bits atomically in a linux PTE, this - * function doesn't need to invalidate tlb. - */ -static inline void radix__ptep_set_access_flags(struct mm_struct *mm, - pte_t *ptep, pte_t entry, - unsigned long address) -{ - - unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED | - _PAGE_RW | _PAGE_EXEC); - - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { - - unsigned long old_pte, new_pte; - - old_pte = __radix_pte_update(ptep, ~0, 0); - /* - * new value of pte - */ - new_pte = old_pte | set; - radix__flush_tlb_pte_p9_dd1(old_pte, mm, address); - __radix_pte_update(ptep, 0, new_pte); - } else - __radix_pte_update(ptep, 0, set); - asm volatile("ptesync" : : : "memory"); -} - static inline int radix__pte_same(pte_t pte_a, pte_t pte_b) { return ((pte_raw(pte_a) ^ pte_raw(pte_b)) == 0); @@ -232,7 +203,24 @@ static inline void radix__set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, int percpu) { *ptep = pte; - asm volatile("ptesync" : : : "memory"); + + /* + * The architecture suggests a ptesync after setting the pte, which + * orders the store that updates the pte with subsequent page table + * walk accesses which may load the pte. Without this it may be + * possible for a subsequent access to result in spurious fault. + * + * This is not necessary for correctness, because a spurious fault + * is tolerated by the page fault handler, and this store will + * eventually be seen. In testing, there was no noticable increase + * in user faults on POWER9. Avoiding ptesync here is a significant + * win for things like fork. If a future microarchitecture benefits + * from ptesync, it should probably go into update_mmu_cache, rather + * than set_pte_at (which is used to set ptes unrelated to faults). + * + * Spurious faults to vmalloc region are not tolerated, so there is + * a ptesync in flush_cache_vmap. + */ } static inline int radix__pmd_bad(pmd_t pmd) diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index 19b45ba6caf9..ef5c3f2994c9 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -51,4 +51,11 @@ extern void radix__flush_tlb_all(void); extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm, unsigned long address); +extern void radix__flush_tlb_lpid_page(unsigned int lpid, + unsigned long addr, + unsigned long page_size); +extern void radix__flush_pwc_lpid(unsigned int lpid); +extern void radix__local_flush_tlb_lpid(unsigned int lpid); +extern void radix__local_flush_tlb_lpid_guest(unsigned int lpid); + #endif diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h index 0cac17253513..ebf572ea621e 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h @@ -4,7 +4,7 @@ #define MMU_NO_CONTEXT ~0UL - +#include <linux/mm_types.h> #include <asm/book3s/64/tlbflush-hash.h> #include <asm/book3s/64/tlbflush-radix.h> @@ -137,6 +137,16 @@ static inline void flush_all_mm(struct mm_struct *mm) #define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr) #define flush_all_mm(mm) local_flush_all_mm(mm) #endif /* CONFIG_SMP */ + +#define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault +static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, + unsigned long address) +{ + /* See ptep_set_access_flags comment */ + if (atomic_read(&vma->vm_mm->context.copros) > 0) + flush_tlb_page(vma, address); +} + /* * flush the page walk cache for the address */ |