diff options
Diffstat (limited to 'arch/mips/kernel')
-rw-r--r-- | arch/mips/kernel/cpu-probe.c | 55 | ||||
-rw-r--r-- | arch/mips/kernel/smp.c | 57 | ||||
-rw-r--r-- | arch/mips/kernel/traps.c | 4 | ||||
-rw-r--r-- | arch/mips/kernel/unaligned.c | 1 |
4 files changed, 110 insertions, 7 deletions
diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index 95b18a194f53..d5e335e6846a 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -872,10 +872,19 @@ static inline unsigned int decode_config4(struct cpuinfo_mips *c) static inline unsigned int decode_config5(struct cpuinfo_mips *c) { - unsigned int config5; + unsigned int config5, max_mmid_width; + unsigned long asid_mask; config5 = read_c0_config5(); config5 &= ~(MIPS_CONF5_UFR | MIPS_CONF5_UFE); + + if (cpu_has_mips_r6) { + if (!__builtin_constant_p(cpu_has_mmid) || cpu_has_mmid) + config5 |= MIPS_CONF5_MI; + else + config5 &= ~MIPS_CONF5_MI; + } + write_c0_config5(config5); if (config5 & MIPS_CONF5_EVA) @@ -894,6 +903,50 @@ static inline unsigned int decode_config5(struct cpuinfo_mips *c) if (config5 & MIPS_CONF5_CRCP) elf_hwcap |= HWCAP_MIPS_CRC32; + if (cpu_has_mips_r6) { + /* Ensure the write to config5 above takes effect */ + back_to_back_c0_hazard(); + + /* Check whether we successfully enabled MMID support */ + config5 = read_c0_config5(); + if (config5 & MIPS_CONF5_MI) + c->options |= MIPS_CPU_MMID; + + /* + * Warn if we've hardcoded cpu_has_mmid to a value unsuitable + * for the CPU we're running on, or if CPUs in an SMP system + * have inconsistent MMID support. + */ + WARN_ON(!!cpu_has_mmid != !!(config5 & MIPS_CONF5_MI)); + + if (cpu_has_mmid) { + write_c0_memorymapid(~0ul); + back_to_back_c0_hazard(); + asid_mask = read_c0_memorymapid(); + + /* + * We maintain a bitmap to track MMID allocation, and + * need a sensible upper bound on the size of that + * bitmap. The initial CPU with MMID support (I6500) + * supports 16 bit MMIDs, which gives us an 8KiB + * bitmap. The architecture recommends that hardware + * support 32 bit MMIDs, which would give us a 512MiB + * bitmap - that's too big in most cases. + * + * Cap MMID width at 16 bits for now & we can revisit + * this if & when hardware supports anything wider. + */ + max_mmid_width = 16; + if (asid_mask > GENMASK(max_mmid_width - 1, 0)) { + pr_info("Capping MMID width at %d bits", + max_mmid_width); + asid_mask = GENMASK(max_mmid_width - 1, 0); + } + + set_cpu_asid_mask(c, asid_mask); + } + } + return config5 & MIPS_CONF_M; } diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index f9dbd95e1d68..6fd9e94fc87e 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -39,6 +39,7 @@ #include <linux/atomic.h> #include <asm/cpu.h> +#include <asm/ginvt.h> #include <asm/processor.h> #include <asm/idle.h> #include <asm/r4k-timer.h> @@ -482,6 +483,15 @@ static void flush_tlb_all_ipi(void *info) void flush_tlb_all(void) { + if (cpu_has_mmid) { + htw_stop(); + ginvt_full(); + sync_ginv(); + instruction_hazard(); + htw_start(); + return; + } + on_each_cpu(flush_tlb_all_ipi, NULL, 1); } @@ -530,7 +540,12 @@ void flush_tlb_mm(struct mm_struct *mm) { preempt_disable(); - if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { + if (cpu_has_mmid) { + /* + * No need to worry about other CPUs - the ginvt in + * drop_mmu_context() will be globalized. + */ + } else if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { smp_on_other_tlbs(flush_tlb_mm_ipi, mm); } else { unsigned int cpu; @@ -561,9 +576,26 @@ static void flush_tlb_range_ipi(void *info) void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { struct mm_struct *mm = vma->vm_mm; + unsigned long addr; + u32 old_mmid; preempt_disable(); - if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { + if (cpu_has_mmid) { + htw_stop(); + old_mmid = read_c0_memorymapid(); + write_c0_memorymapid(cpu_asid(0, mm)); + mtc0_tlbw_hazard(); + addr = round_down(start, PAGE_SIZE * 2); + end = round_up(end, PAGE_SIZE * 2); + do { + ginvt_va_mmid(addr); + sync_ginv(); + addr += PAGE_SIZE * 2; + } while (addr < end); + write_c0_memorymapid(old_mmid); + instruction_hazard(); + htw_start(); + } else if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { struct flush_tlb_data fd = { .vma = vma, .addr1 = start, @@ -571,6 +603,7 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l }; smp_on_other_tlbs(flush_tlb_range_ipi, &fd); + local_flush_tlb_range(vma, start, end); } else { unsigned int cpu; int exec = vma->vm_flags & VM_EXEC; @@ -585,8 +618,8 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l if (cpu != smp_processor_id() && cpu_context(cpu, mm)) set_cpu_context(cpu, mm, !exec); } + local_flush_tlb_range(vma, start, end); } - local_flush_tlb_range(vma, start, end); preempt_enable(); } @@ -616,14 +649,28 @@ static void flush_tlb_page_ipi(void *info) void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) { + u32 old_mmid; + preempt_disable(); - if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { + if (cpu_has_mmid) { + htw_stop(); + old_mmid = read_c0_memorymapid(); + write_c0_memorymapid(cpu_asid(0, vma->vm_mm)); + mtc0_tlbw_hazard(); + ginvt_va_mmid(page); + sync_ginv(); + write_c0_memorymapid(old_mmid); + instruction_hazard(); + htw_start(); + } else if ((atomic_read(&vma->vm_mm->mm_users) != 1) || + (current->mm != vma->vm_mm)) { struct flush_tlb_data fd = { .vma = vma, .addr1 = page, }; smp_on_other_tlbs(flush_tlb_page_ipi, &fd); + local_flush_tlb_page(vma, page); } else { unsigned int cpu; @@ -637,8 +684,8 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) set_cpu_context(cpu, vma->vm_mm, 1); } + local_flush_tlb_page(vma, page); } - local_flush_tlb_page(vma, page); preempt_enable(); } diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index c91097f7b32f..995249be64f1 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -2223,7 +2223,9 @@ void per_cpu_trap_init(bool is_boot_cpu) cp0_fdc_irq = -1; } - if (!cpu_data[cpu].asid_cache) + if (cpu_has_mmid) + cpu_data[cpu].asid_cache = 0; + else if (!cpu_data[cpu].asid_cache) cpu_data[cpu].asid_cache = asid_first_version(cpu); mmgrab(&init_mm); diff --git a/arch/mips/kernel/unaligned.c b/arch/mips/kernel/unaligned.c index 0ed20a64b285..76e33f940971 100644 --- a/arch/mips/kernel/unaligned.c +++ b/arch/mips/kernel/unaligned.c @@ -89,6 +89,7 @@ #include <asm/fpu.h> #include <asm/fpu_emulator.h> #include <asm/inst.h> +#include <asm/mmu_context.h> #include <linux/uaccess.h> #define STR(x) __STR(x) |