From 026b5ca3b65f878019a8eb0c7a702cd5c20a4104 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Wed, 1 Sep 2010 14:33:29 +0100 Subject: ARM: 6344/1: Mark CPU_32v6K as depended on CPU_V7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CPU_32v6K is selected by CPU_V7 but it only depends on CPU_V6. Signed-off-by: Catalin Marinas Acked-by: Uwe Kleine-König Signed-off-by: Russell King --- arch/arm/mm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 33c3f570aaa0..a0a2928ae4dd 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -398,7 +398,7 @@ config CPU_V6 # ARMv6k config CPU_32v6K bool "Support ARM V6K processor extensions" if !SMP - depends on CPU_V6 + depends on CPU_V6 || CPU_V7 default y if SMP && !(ARCH_MX3 || ARCH_OMAP2) help Say Y here if your ARMv6 processor supports the 'K' extension. -- cgit v1.2.3 From 3a4b5dca53aecb16db9e007d782b2d1e757e941a Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 3 Sep 2010 10:39:59 +0100 Subject: ARM: 6355/1: hw-breakpoint: add mechanism for hooking into prefetch aborts On ARM processors with hardware breakpoint and watchpoint support, triggering these events results in a debug exception. These manifest as prefetch and data aborts respectively. arch/arm/mm/fault.c already provides hook_fault_code for hooking into data aborts dependent on the DFSR. This patch adds a new function, hook_ifault_code for hooking into prefetch aborts in the same manner. Cc: Frederic Weisbecker Cc: S. Karthikeyan Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/mm/fault.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 23b0b03af5ea..1e21e125fe3a 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -581,6 +581,19 @@ static struct fsr_info ifsr_info[] = { { do_bad, SIGBUS, 0, "unknown 31" }, }; +void __init +hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), + int sig, int code, const char *name) +{ + if (nr < 0 || nr >= ARRAY_SIZE(ifsr_info)) + BUG(); + + ifsr_info[nr].fn = fn; + ifsr_info[nr].sig = sig; + ifsr_info[nr].code = code; + ifsr_info[nr].name = name; +} + asmlinkage void __exception do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) { -- cgit v1.2.3 From 2be23c475af8ae4e25f8bab08d815b17593bd547 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 8 Sep 2010 16:27:56 +0100 Subject: ARM: Ensure PTE modifications via dma_alloc_coherent are visible Dave Hylands reports: | We've observed a problem with dma_alloc_writecombine when the system | is under heavy load (heavy bus traffic). We've managed to reduce the | problem to the following snippet, which is run from a kthread in a | continuous loop: | | void *virtAddr; | dma_addr_t physAddr; | unsigned int numBytes = 256; | | for (;;) { | virtAddr = dma_alloc_writecombine(NULL, | numBytes, &physAddr, GFP_KERNEL); | if (virtAddr == NULL) { | printk(KERN_ERR "Running out of memory\n"); | break; | } | | /* access DMA memory allocated */ | tmp = virtAddr; | *tmp = 0x77; | | /* free DMA memory */ | dma_free_writecombine(NULL, | numBytes, virtAddr, physAddr); | | ...sleep here... | } | | By itself, the code will run forever with no issues. However, as we | increase our bus traffic (typically using DMA) then the *tmp = 0x77 | line will eventually cause a page fault. If we add a small delay (a | few microseconds) before the *tmp = 0x77, then we don't see a page | fault, even under heavy load. A dsb() is required after modifying the PTE entries to ensure that they will always be visible. Add this dsb(). Reported-by: Dave Hylands Tested-by: Dave Hylands Signed-off-by: Russell King --- arch/arm/mm/dma-mapping.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index c704eed63c5d..4bc43e535d3b 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -229,6 +229,8 @@ __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot) } } while (size -= PAGE_SIZE); + dsb(); + return (void *)c->vm_start; } return NULL; -- cgit v1.2.3 From 6491848d1ab246f6d243ddef25085fc1d836ff2c Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 14 Sep 2010 09:50:03 +0100 Subject: ARM: 6387/1: errata: check primary part ID in proc-v7.S Kconfig doesn't have any knowledge of specific v7 cores, so it is possible to select errata workarounds that may cause inadvertent behaviour when executed on a core other than those targetted by the fix. This patch improves the variant and revision checking in proc-v7.S so that the primary part number is also considered when applying errata workarounds. Acked-by: Catalin Marinas Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/mm/proc-v7.S | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 6a8506d99ee9..1f16f9e3f441 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -204,8 +204,13 @@ __v7_setup: bne 2f and r5, r0, #0x00f00000 @ variant and r6, r0, #0x0000000f @ revision - orr r0, r6, r5, lsr #20-4 @ combine variant and revision + orr r6, r6, r5, lsr #20-4 @ combine variant and revision + ubfx r0, r0, #4, #12 @ primary part number + /* Cortex-A8 Errata */ + ldr r10, =0x00000c08 @ Cortex-A8 primary part number + teq r0, r10 + bne 2f #ifdef CONFIG_ARM_ERRATA_430973 teq r5, #0x00100000 @ only present in r1p* mrceq p15, 0, r10, c1, c0, 1 @ read aux control register @@ -213,14 +218,14 @@ __v7_setup: mcreq p15, 0, r10, c1, c0, 1 @ write aux control register #endif #ifdef CONFIG_ARM_ERRATA_458693 - teq r0, #0x20 @ only present in r2p0 + teq r6, #0x20 @ only present in r2p0 mrceq p15, 0, r10, c1, c0, 1 @ read aux control register orreq r10, r10, #(1 << 5) @ set L1NEON to 1 orreq r10, r10, #(1 << 9) @ set PLDNOP to 1 mcreq p15, 0, r10, c1, c0, 1 @ write aux control register #endif #ifdef CONFIG_ARM_ERRATA_460075 - teq r0, #0x20 @ only present in r2p0 + teq r6, #0x20 @ only present in r2p0 mrceq p15, 1, r10, c9, c0, 2 @ read L2 cache aux ctrl register tsteq r10, #1 << 22 orreq r10, r10, #(1 << 22) @ set the Write Allocate disable bit -- cgit v1.2.3 From 9f05027c7cb3cfe56a31892bd83391138d41a667 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 14 Sep 2010 09:51:43 +0100 Subject: ARM: 6388/1: errata: DMB operation may be faulty On versions of the Cortex-A9 up to and including r2p2, under rare circumstances, a DMB instruction between 2 write operations may not ensure the correct visibility ordering of the 2 writes. This workaround sets a bit in the diagnostic register of the Cortex-A9, causing the DMB instruction to behave like a DSB, which functions correctly on the affected cores. Acked-by: Catalin Marinas Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/mm/proc-v7.S | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 1f16f9e3f441..945f36341fa6 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -201,7 +201,7 @@ __v7_setup: mrc p15, 0, r0, c0, c0, 0 @ read main ID register and r10, r0, #0xff000000 @ ARM? teq r10, #0x41000000 - bne 2f + bne 3f and r5, r0, #0x00f00000 @ variant and r6, r0, #0x0000000f @ revision orr r6, r6, r5, lsr #20-4 @ combine variant and revision @@ -231,8 +231,20 @@ __v7_setup: orreq r10, r10, #(1 << 22) @ set the Write Allocate disable bit mcreq p15, 1, r10, c9, c0, 2 @ write the L2 cache aux ctrl register #endif + b 3f + + /* Cortex-A9 Errata */ +2: ldr r10, =0x00000c09 @ Cortex-A9 primary part number + teq r0, r10 + bne 3f +#ifdef CONFIG_ARM_ERRATA_742230 + cmp r6, #0x22 @ only present up to r2p2 + mrcle p15, 0, r10, c15, c0, 1 @ read diagnostic register + orrle r10, r10, #1 << 4 @ set bit #4 + mcrle p15, 0, r10, c15, c0, 1 @ write diagnostic register +#endif -2: mov r10, #0 +3: mov r10, #0 #ifdef HARVARD_CACHE mcr p15, 0, r10, c7, c5, 0 @ I+BTB cache invalidate #endif -- cgit v1.2.3 From a672e99b129e286df2e2697a1b603d82321117f3 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 14 Sep 2010 09:53:02 +0100 Subject: ARM: 6389/1: errata: incorrect hazard handling in the SCU may lead to data corruption On the r2p0, r2p1 and r2p2 versions of the Cortex-A9, data corruption can occur if a shared cache line is replaced on one CPU as another CPU is accessing it. This workaround sets two bits in the diagnostic register of the Cortex-A9, reducing the linefill issuing capabilities of the processor and avoiding the erroneous behaviour. Acked-by: Catalin Marinas Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/mm/proc-v7.S | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 945f36341fa6..080129263eef 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -243,6 +243,15 @@ __v7_setup: orrle r10, r10, #1 << 4 @ set bit #4 mcrle p15, 0, r10, c15, c0, 1 @ write diagnostic register #endif +#ifdef CONFIG_ARM_ERRATA_742231 + teq r6, #0x20 @ present in r2p0 + teqne r6, #0x21 @ present in r2p1 + teqne r6, #0x22 @ present in r2p2 + mrceq p15, 0, r10, c15, c0, 1 @ read diagnostic register + orreq r10, r10, #1 << 12 @ set bit #12 + orreq r10, r10, #1 << 22 @ set bit #22 + mcreq p15, 0, r10, c15, c0, 1 @ write diagnostic register +#endif 3: mov r10, #0 #ifdef HARVARD_CACHE -- cgit v1.2.3 From 14eff1812679c76564b775aa95cdd378965f6cfb Mon Sep 17 00:00:00 2001 From: Daniel Walker Date: Fri, 17 Sep 2010 16:42:10 +0100 Subject: ARM: 6398/1: add proc info for ARM11MPCore/Cortex-A9 from ARM Setting of these bits can cause issues on other SMP SoC's not produced by ARM. Acked-by: Catalin Marinas Signed-off-by: Daniel Walker Signed-off-by: Russell King --- arch/arm/mm/proc-v7.S | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 080129263eef..7563ff0141bd 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -186,13 +186,14 @@ cpu_v7_name: * It is assumed that: * - cache type register is implemented */ -__v7_setup: +__v7_ca9mp_setup: #ifdef CONFIG_SMP mrc p15, 0, r0, c1, c0, 1 tst r0, #(1 << 6) @ SMP/nAMP mode enabled? orreq r0, r0, #(1 << 6) | (1 << 0) @ Enable SMP/nAMP mode and mcreq p15, 0, r0, c1, c0, 1 @ TLB ops broadcasting #endif +__v7_setup: adr r12, __v7_setup_stack @ the local stack stmia r12, {r0-r5, r7, r9, r11, lr} bl v7_flush_dcache_all @@ -349,6 +350,29 @@ cpu_elf_name: .section ".proc.info.init", #alloc, #execinstr + .type __v7_ca9mp_proc_info, #object +__v7_ca9mp_proc_info: + .long 0x410fc090 @ Required ID value + .long 0xff0ffff0 @ Mask for ID + .long PMD_TYPE_SECT | \ + PMD_SECT_AP_WRITE | \ + PMD_SECT_AP_READ | \ + PMD_FLAGS + .long PMD_TYPE_SECT | \ + PMD_SECT_XN | \ + PMD_SECT_AP_WRITE | \ + PMD_SECT_AP_READ + b __v7_ca9mp_setup + .long cpu_arch_name + .long cpu_elf_name + .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP + .long cpu_v7_name + .long v7_processor_functions + .long v7wbi_tlb_fns + .long v6_user_fns + .long v7_cache_fns + .size __v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info + /* * Match any ARMv7 processor core. */ -- cgit v1.2.3 From 0fc73099dd25df2c5181b7bad57d1faa5cd12d3c Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 13 Sep 2010 15:57:05 +0100 Subject: ARM: 6378/1: Allow lazy cache flushing via PG_arch_1 for highmem pages Commit d73cd42 forced non-lazy cache flushing of highmem pages in flush_dcache_page(). This isn't needed since __flush_dcache_page() (called lazily from update_mmu_cache) can handle highmem pages (fixed by commit 7e5a69e). Signed-off-by: Catalin Marinas Acked-by: Nicolas Pitre Signed-off-by: Russell King --- arch/arm/mm/flush.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index c6844cb9b508..87dd5ffdfa2d 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -247,7 +247,7 @@ void flush_dcache_page(struct page *page) mapping = page_mapping(page); #ifndef CONFIG_SMP - if (!PageHighMem(page) && mapping && !mapping_mapped(mapping)) + if (mapping && !mapping_mapped(mapping)) set_bit(PG_dcache_dirty, &page->flags); else #endif -- cgit v1.2.3 From c01778001a4f5ad9c62d882776235f3f31922fdd Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 13 Sep 2010 15:57:36 +0100 Subject: ARM: 6379/1: Assume new page cache pages have dirty D-cache There are places in Linux where writes to newly allocated page cache pages happen without a subsequent call to flush_dcache_page() (several PIO drivers including USB HCD). This patch changes the meaning of PG_arch_1 to be PG_dcache_clean and always flush the D-cache for a newly mapped page in update_mmu_cache(). The patch also sets the PG_arch_1 bit in the DMA cache maintenance function to avoid additional cache flushing in update_mmu_cache(). Tested-by: Rabin Vincent Cc: Nicolas Pitre Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/copypage-v4mc.c | 2 +- arch/arm/mm/copypage-v6.c | 2 +- arch/arm/mm/copypage-xscale.c | 2 +- arch/arm/mm/dma-mapping.c | 6 ++++++ arch/arm/mm/fault-armv.c | 4 ++-- arch/arm/mm/flush.c | 3 ++- 6 files changed, 13 insertions(+), 6 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c index 598c51ad5071..b8061519ce77 100644 --- a/arch/arm/mm/copypage-v4mc.c +++ b/arch/arm/mm/copypage-v4mc.c @@ -73,7 +73,7 @@ void v4_mc_copy_user_highpage(struct page *to, struct page *from, { void *kto = kmap_atomic(to, KM_USER1); - if (test_and_clear_bit(PG_dcache_dirty, &from->flags)) + if (!test_and_set_bit(PG_dcache_clean, &from->flags)) __flush_dcache_page(page_mapping(from), from); spin_lock(&minicache_lock); diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c index f55fa1044f72..bdba6c65c901 100644 --- a/arch/arm/mm/copypage-v6.c +++ b/arch/arm/mm/copypage-v6.c @@ -79,7 +79,7 @@ static void v6_copy_user_highpage_aliasing(struct page *to, unsigned int offset = CACHE_COLOUR(vaddr); unsigned long kfrom, kto; - if (test_and_clear_bit(PG_dcache_dirty, &from->flags)) + if (!test_and_set_bit(PG_dcache_clean, &from->flags)) __flush_dcache_page(page_mapping(from), from); /* FIXME: not highmem safe */ diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c index 9920c0ae2096..649bbcd325bf 100644 --- a/arch/arm/mm/copypage-xscale.c +++ b/arch/arm/mm/copypage-xscale.c @@ -95,7 +95,7 @@ void xscale_mc_copy_user_highpage(struct page *to, struct page *from, { void *kto = kmap_atomic(to, KM_USER1); - if (test_and_clear_bit(PG_dcache_dirty, &from->flags)) + if (!test_and_set_bit(PG_dcache_clean, &from->flags)) __flush_dcache_page(page_mapping(from), from); spin_lock(&minicache_lock); diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 4bc43e535d3b..e4dd0646e859 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -523,6 +523,12 @@ void ___dma_page_dev_to_cpu(struct page *page, unsigned long off, outer_inv_range(paddr, paddr + size); dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); + + /* + * Mark the D-cache clean for this page to avoid extra flushing. + */ + if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE) + set_bit(PG_dcache_clean, &page->flags); } EXPORT_SYMBOL(___dma_page_dev_to_cpu); diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 9b906dec1ca1..58846cbd0e0b 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -141,7 +141,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, * a page table, or changing an existing PTE. Basically, there are two * things that we need to take care of: * - * 1. If PG_dcache_dirty is set for the page, we need to ensure + * 1. If PG_dcache_clean is not set for the page, we need to ensure * that any cache entries for the kernels virtual memory * range are written back to the page. * 2. If we have multiple shared mappings of the same space in @@ -169,7 +169,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, mapping = page_mapping(page); #ifndef CONFIG_SMP - if (test_and_clear_bit(PG_dcache_dirty, &page->flags)) + if (!test_and_set_bit(PG_dcache_clean, &page->flags)) __flush_dcache_page(mapping, page); #endif if (mapping) { diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 87dd5ffdfa2d..b4efce9b7985 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -248,7 +248,7 @@ void flush_dcache_page(struct page *page) #ifndef CONFIG_SMP if (mapping && !mapping_mapped(mapping)) - set_bit(PG_dcache_dirty, &page->flags); + clear_bit(PG_dcache_clean, &page->flags); else #endif { @@ -257,6 +257,7 @@ void flush_dcache_page(struct page *page) __flush_dcache_aliases(mapping, page); else if (mapping) __flush_icache_all(); + set_bit(PG_dcache_clean, &page->flags); } } EXPORT_SYMBOL(flush_dcache_page); -- cgit v1.2.3 From 6012191aa9c6ffff3a23b81162298318b56d7cb3 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 13 Sep 2010 15:58:06 +0100 Subject: ARM: 6380/1: Introduce __sync_icache_dcache() for VIPT caches On SMP systems, there is a small chance of a PTE becoming visible to a different CPU before the current cache maintenance operations in update_mmu_cache(). To avoid this, cache maintenance must be handled in set_pte_at() (similar to IA-64 and PowerPC). This patch provides a unified VIPT cache handling mechanism and implements the __sync_icache_dcache() function for ARMv6 onwards architectures. It is called from set_pte_at() and replaces the update_mmu_cache(). The latter is still used on VIVT hardware where a vm_area_struct is required. Tested-by: Rabin Vincent Cc: Nicolas Pitre Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 4 ++-- arch/arm/mm/flush.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 58846cbd0e0b..8440d952ba6d 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -28,6 +28,7 @@ static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE; +#if __LINUX_ARM_ARCH__ < 6 /* * We take the easy way out of this problem - we make the * PTE uncacheable. However, we leave the write buffer on. @@ -168,10 +169,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, return; mapping = page_mapping(page); -#ifndef CONFIG_SMP if (!test_and_set_bit(PG_dcache_clean, &page->flags)) __flush_dcache_page(mapping, page); -#endif if (mapping) { if (cache_is_vivt()) make_coherent(mapping, vma, addr, ptep, pfn); @@ -179,6 +178,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, __flush_icache_all(); } } +#endif /* __LINUX_ARM_ARCH__ < 6 */ /* * Check whether the write buffer has physical address aliasing diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index b4efce9b7985..dd5b0120b92e 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -215,6 +215,36 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct page *p flush_dcache_mmap_unlock(mapping); } +#if __LINUX_ARM_ARCH__ >= 6 +void __sync_icache_dcache(pte_t pteval) +{ + unsigned long pfn; + struct page *page; + struct address_space *mapping; + + if (!pte_present_user(pteval)) + return; + if (cache_is_vipt_nonaliasing() && !pte_exec(pteval)) + /* only flush non-aliasing VIPT caches for exec mappings */ + return; + pfn = pte_pfn(pteval); + if (!pfn_valid(pfn)) + return; + + page = pfn_to_page(pfn); + if (cache_is_vipt_aliasing()) + mapping = page_mapping(page); + else + mapping = NULL; + + if (!test_and_set_bit(PG_dcache_clean, &page->flags)) + __flush_dcache_page(mapping, page); + /* pte_exec() already checked above for non-aliasing VIPT cache */ + if (cache_is_vipt_nonaliasing() || pte_exec(pteval)) + __flush_icache_all(); +} +#endif + /* * Ensure cache coherency between kernel mapping and userspace mapping * of this page. -- cgit v1.2.3 From 85848dd7ab75fce1134856228582a8df522c91d9 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 13 Sep 2010 15:58:37 +0100 Subject: ARM: 6381/1: Use lazy cache flushing on ARMv7 SMP systems ARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. This patch allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case. Note that cache flushing on SMP systems now takes place via the set_pte_at() call (__sync_icache_dcache) and there is no race with other CPUs executing code from the new PTE before the cache flushing took place. Tested-by: Rabin Vincent Cc: Nicolas Pitre Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/flush.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index dd5b0120b92e..2332b774c6b9 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "mm.h" @@ -93,12 +94,10 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig #define flush_pfn_alias(pfn,vaddr) do { } while (0) #endif -#ifdef CONFIG_SMP static void flush_ptrace_access_other(void *args) { __flush_icache_all(); } -#endif static void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, @@ -122,11 +121,9 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, if (vma->vm_flags & VM_EXEC) { unsigned long addr = (unsigned long)kaddr; __cpuc_coherent_kern_range(addr, addr + len); -#ifdef CONFIG_SMP if (cache_ops_need_broadcast()) smp_call_function(flush_ptrace_access_other, NULL, 1); -#endif } } @@ -276,12 +273,10 @@ void flush_dcache_page(struct page *page) mapping = page_mapping(page); -#ifndef CONFIG_SMP - if (mapping && !mapping_mapped(mapping)) + if (!cache_ops_need_broadcast() && + mapping && !mapping_mapped(mapping)) clear_bit(PG_dcache_clean, &page->flags); - else -#endif - { + else { __flush_dcache_page(mapping, page); if (mapping && cache_is_vivt()) __flush_dcache_aliases(mapping, page); -- cgit v1.2.3 From d907387c42e9e39261629890e45a08ef4c3ed3fe Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 13 Sep 2010 16:01:24 +0100 Subject: ARM: 6383/1: Implement phys_mem_access_prot() to avoid attributes aliasing ARMv7 onwards requires that there are no aliases to the same physical location using different memory types (i.e. Normal vs Strongly Ordered). Access to SO mappings when the unaligned accesses are handled in hardware is also Unpredictable (pgprot_noncached() mappings in user space). The /dev/mem driver requires uncached mappings with O_SYNC. The patch implements the phys_mem_access_prot() function which generates Strongly Ordered memory attributes if !pfn_valid() (independent of O_SYNC) and Normal Noncacheable (writecombine) if O_SYNC. Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/mmu.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 6e1c4f6a2b3f..a486bd0d97dc 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -498,6 +499,19 @@ static void __init build_mem_type_table(void) } } +#ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE +pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, + unsigned long size, pgprot_t vma_prot) +{ + if (!pfn_valid(pfn)) + return pgprot_noncached(vma_prot); + else if (file->f_flags & O_SYNC) + return pgprot_writecombine(vma_prot); + return vma_prot; +} +EXPORT_SYMBOL(phys_mem_access_prot); +#endif + #define vectors_base() (vectors_high() ? 0xffff0000 : 0) static void __init *early_alloc(unsigned long sz) -- cgit v1.2.3 From 2f27bf834e1d0a06e83d7458b535891c552271aa Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Mon, 20 Sep 2010 04:10:43 +0100 Subject: ARM: 6401/1: plug a race in the alignment trap handler When the policy for user space is to ignore misaligned accesses from user space, the processor then performs a documented rotation on the accessed data. This is the result of the access being trapped, and the kernel disabling the alignment trap before returning to user space again. In kernel space we always want misaligned accesses to be fixed up. This is enforced by always re-enabling the alignment trap on every entry into kernel space from user space. No such re-enabling is performed when an exception occurs while already in kernel space as the alignment trap is always supposed to be enabled in that case. There is however a small race window when a misaligned access in user space is trapped and the alignment trap disabled, but the CPU didn't return to user space just yet. Any exception would be entered from kernel space at that point and the kernel would then execute with the alignment trap disabled. Thanks to Maxime Bizon for providing a test module that made this issue reproducible. Signed-off-by: Nicolas Pitre Signed-off-by: Russell King --- arch/arm/mm/alignment.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index d073b64ae87e..724ba3bce72c 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -885,8 +885,23 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs) if (ai_usermode & UM_SIGNAL) force_sig(SIGBUS, current); - else - set_cr(cr_no_alignment); + else { + /* + * We're about to disable the alignment trap and return to + * user space. But if an interrupt occurs before actually + * reaching user space, then the IRQ vector entry code will + * notice that we were still in kernel space and therefore + * the alignment trap won't be re-enabled in that case as it + * is presumed to be always on from kernel space. + * Let's prevent that race by disabling interrupts here (they + * are disabled on the way back to user space anyway in + * entry-common.S) and disable the alignment trap only if + * there is no work pending for this thread. + */ + raw_local_irq_disable(); + if (!(current_thread_info()->flags & _TIF_WORK_MASK)) + set_cr(cr_no_alignment); + } return 0; } -- cgit v1.2.3 From f1a2481c0ad3aebd94d11b317c488deaadc25002 Mon Sep 17 00:00:00 2001 From: Santosh Shilimkar Date: Fri, 24 Sep 2010 07:18:22 +0100 Subject: ARM: 6407/1: mmu: Setup MT_MEMORY and MT_MEMORY_NONCACHED L1 entries This patch populates the L1 entries for MT_MEMORY and MT_MEMORY_NONCACHED types so that at boot-up, we can map memories outside system memory at page level granularity Previously the mapping was limiting to section level, which creates unnecessary additional mapping for which physical memory may not present. On the newer ARM with speculation, this is dangerous and can result in untraceable aborts. Signed-off-by: Santosh Shilimkar Signed-off-by: Russell King --- arch/arm/mm/mmu.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index a486bd0d97dc..6a3a2d0cd6db 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -247,6 +247,9 @@ static struct mem_type mem_types[] = { .domain = DOMAIN_USER, }, [MT_MEMORY] = { + .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | + L_PTE_USER | L_PTE_EXEC, + .prot_l1 = PMD_TYPE_TABLE, .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE, .domain = DOMAIN_KERNEL, }, @@ -255,6 +258,9 @@ static struct mem_type mem_types[] = { .domain = DOMAIN_KERNEL, }, [MT_MEMORY_NONCACHED] = { + .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | + L_PTE_USER | L_PTE_EXEC | L_PTE_MT_BUFFERABLE, + .prot_l1 = PMD_TYPE_TABLE, .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE, .domain = DOMAIN_KERNEL, }, @@ -412,9 +418,12 @@ static void __init build_mem_type_table(void) * Enable CPU-specific coherency if supported. * (Only available on XSC3 at the moment.) */ - if (arch_is_coherent() && cpu_is_xsc3()) + if (arch_is_coherent() && cpu_is_xsc3()) { mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; - + mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED; + mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S; + mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED; + } /* * ARMv6 and above have extended page tables. */ @@ -439,7 +448,9 @@ static void __init build_mem_type_table(void) mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S; mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED; mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; + mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED; mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S; + mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED; #endif } @@ -476,6 +487,8 @@ static void __init build_mem_type_table(void) mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask; mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask; mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd; + mem_types[MT_MEMORY].prot_pte |= kern_pgprot; + mem_types[MT_MEMORY_NONCACHED].prot_sect |= ecc_mask; mem_types[MT_ROM].prot_sect |= cp->pmd; switch (cp->pmd) { -- cgit v1.2.3 From f00ec48fadf5e37e7889f14cff900aa70d18b644 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 4 Sep 2010 10:47:48 +0100 Subject: ARM: Allow SMP kernels to boot on UP systems UP systems do not implement all the instructions that SMP systems have, so in order to boot a SMP kernel on a UP system, we need to rewrite parts of the kernel. Do this using an 'alternatives' scheme, where the kernel code and data is modified prior to initialization to replace the SMP instructions, thereby rendering the problematical code ineffectual. We use the linker to generate a list of 32-bit word locations and their replacement values, and run through these replacements when we detect a UP system. Signed-off-by: Russell King --- arch/arm/mm/cache-v7.S | 14 ++++---------- arch/arm/mm/mmu.c | 46 +++++++++++++++++++++------------------------- arch/arm/mm/proc-v6.S | 43 ++++++++++++++++++++++++++++--------------- arch/arm/mm/proc-v7.S | 41 +++++++++++++++++++++++++++-------------- arch/arm/mm/tlb-v7.S | 33 ++++++++++++--------------------- 5 files changed, 92 insertions(+), 85 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index 37c8157e116e..e8ea1a071f43 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -91,11 +91,8 @@ ENTRY(v7_flush_kern_cache_all) THUMB( stmfd sp!, {r4-r7, r9-r11, lr} ) bl v7_flush_dcache_all mov r0, #0 -#ifdef CONFIG_SMP - mcr p15, 0, r0, c7, c1, 0 @ invalidate I-cache inner shareable -#else - mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate -#endif + ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable + ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} ) THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} ) mov pc, lr @@ -171,11 +168,8 @@ ENTRY(v7_coherent_user_range) cmp r0, r1 blo 1b mov r0, #0 -#ifdef CONFIG_SMP - mcr p15, 0, r0, c7, c1, 6 @ invalidate BTB Inner Shareable -#else - mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB -#endif + ALT_SMP(mcr p15, 0, r0, c7, c1, 6) @ invalidate BTB Inner Shareable + ALT_UP(mcr p15, 0, r0, c7, c5, 6) @ invalidate BTB dsb isb mov pc, lr diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 6a3a2d0cd6db..e2335811c02e 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -310,9 +310,8 @@ static void __init build_mem_type_table(void) cachepolicy = CPOLICY_WRITEBACK; ecc_mask = 0; } -#ifdef CONFIG_SMP - cachepolicy = CPOLICY_WRITEALLOC; -#endif + if (is_smp()) + cachepolicy = CPOLICY_WRITEALLOC; /* * Strip out features not present on earlier architectures. @@ -406,13 +405,11 @@ static void __init build_mem_type_table(void) cp = &cache_policies[cachepolicy]; vecs_pgprot = kern_pgprot = user_pgprot = cp->pte; -#ifndef CONFIG_SMP /* * Only use write-through for non-SMP systems */ - if (cpu_arch >= CPU_ARCH_ARMv5 && cachepolicy > CPOLICY_WRITETHROUGH) + if (!is_smp() && cpu_arch >= CPU_ARCH_ARMv5 && cachepolicy > CPOLICY_WRITETHROUGH) vecs_pgprot = cache_policies[CPOLICY_WRITETHROUGH].pte; -#endif /* * Enable CPU-specific coherency if supported. @@ -436,22 +433,23 @@ static void __init build_mem_type_table(void) mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; -#ifdef CONFIG_SMP - /* - * Mark memory with the "shared" attribute for SMP systems - */ - user_pgprot |= L_PTE_SHARED; - kern_pgprot |= L_PTE_SHARED; - vecs_pgprot |= L_PTE_SHARED; - mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S; - mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED; - mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S; - mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED; - mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; - mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED; - mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S; - mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED; -#endif + if (is_smp()) { + /* + * Mark memory with the "shared" attribute + * for SMP systems + */ + user_pgprot |= L_PTE_SHARED; + kern_pgprot |= L_PTE_SHARED; + vecs_pgprot |= L_PTE_SHARED; + mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S; + mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED; + mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S; + mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED; + mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; + mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED; + mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S; + mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED; + } } /* @@ -829,8 +827,7 @@ static void __init sanity_check_meminfo(void) * rather difficult. */ reason = "with VIPT aliasing cache"; -#ifdef CONFIG_SMP - } else if (tlb_ops_need_broadcast()) { + } else if (is_smp() && tlb_ops_need_broadcast()) { /* * kmap_high needs to occasionally flush TLB entries, * however, if the TLB entries need to be broadcast @@ -840,7 +837,6 @@ static void __init sanity_check_meminfo(void) * (must not be called with irqs off) */ reason = "without hardware TLB ops broadcasting"; -#endif } if (reason) { printk(KERN_CRIT "HIGHMEM is not supported %s, ignoring high memory\n", diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S index 22aac8515196..b95662dedb64 100644 --- a/arch/arm/mm/proc-v6.S +++ b/arch/arm/mm/proc-v6.S @@ -30,13 +30,10 @@ #define TTB_RGN_WT (2 << 3) #define TTB_RGN_WB (3 << 3) -#ifndef CONFIG_SMP -#define TTB_FLAGS TTB_RGN_WBWA -#define PMD_FLAGS PMD_SECT_WB -#else -#define TTB_FLAGS TTB_RGN_WBWA|TTB_S -#define PMD_FLAGS PMD_SECT_WBWA|PMD_SECT_S -#endif +#define TTB_FLAGS_UP TTB_RGN_WBWA +#define PMD_FLAGS_UP PMD_SECT_WB +#define TTB_FLAGS_SMP TTB_RGN_WBWA|TTB_S +#define PMD_FLAGS_SMP PMD_SECT_WBWA|PMD_SECT_S ENTRY(cpu_v6_proc_init) mov pc, lr @@ -97,7 +94,8 @@ ENTRY(cpu_v6_switch_mm) #ifdef CONFIG_MMU mov r2, #0 ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id - orr r0, r0, #TTB_FLAGS + ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP) + ALT_UP(orr r0, r0, #TTB_FLAGS_UP) mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB mcr p15, 0, r2, c7, c10, 4 @ drain write buffer mcr p15, 0, r0, c2, c0, 0 @ set TTB 0 @@ -156,9 +154,11 @@ cpu_pj4_name: */ __v6_setup: #ifdef CONFIG_SMP - mrc p15, 0, r0, c1, c0, 1 @ Enable SMP/nAMP mode + ALT_SMP(mrc p15, 0, r0, c1, c0, 1) @ Enable SMP/nAMP mode + ALT_UP(nop) orr r0, r0, #0x20 - mcr p15, 0, r0, c1, c0, 1 + ALT_SMP(mcr p15, 0, r0, c1, c0, 1) + ALT_UP(nop) #endif mov r0, #0 @@ -169,7 +169,8 @@ __v6_setup: #ifdef CONFIG_MMU mcr p15, 0, r0, c8, c7, 0 @ invalidate I + D TLBs mcr p15, 0, r0, c2, c0, 2 @ TTB control register - orr r4, r4, #TTB_FLAGS + ALT_SMP(orr r4, r4, #TTB_FLAGS_SMP) + ALT_UP(orr r4, r4, #TTB_FLAGS_UP) mcr p15, 0, r4, c2, c0, 1 @ load TTB1 #endif /* CONFIG_MMU */ adr r5, v6_crval @@ -225,10 +226,16 @@ cpu_elf_name: __v6_proc_info: .long 0x0007b000 .long 0x0007f000 - .long PMD_TYPE_SECT | \ + ALT_SMP(.long \ + PMD_TYPE_SECT | \ + PMD_SECT_AP_WRITE | \ + PMD_SECT_AP_READ | \ + PMD_FLAGS_SMP) + ALT_UP(.long \ + PMD_TYPE_SECT | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ | \ - PMD_FLAGS + PMD_FLAGS_UP) .long PMD_TYPE_SECT | \ PMD_SECT_XN | \ PMD_SECT_AP_WRITE | \ @@ -249,10 +256,16 @@ __v6_proc_info: __pj4_v6_proc_info: .long 0x560f5810 .long 0xff0ffff0 - .long PMD_TYPE_SECT | \ + ALT_SMP(.long \ + PMD_TYPE_SECT | \ + PMD_SECT_AP_WRITE | \ + PMD_SECT_AP_READ | \ + PMD_FLAGS_SMP) + ALT_UP(.long \ + PMD_TYPE_SECT | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ | \ - PMD_FLAGS + PMD_FLAGS_UP) .long PMD_TYPE_SECT | \ PMD_SECT_XN | \ PMD_SECT_AP_WRITE | \ diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 7563ff0141bd..df422fee1cb6 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -30,15 +30,13 @@ #define TTB_IRGN_WT ((1 << 0) | (0 << 6)) #define TTB_IRGN_WB ((1 << 0) | (1 << 6)) -#ifndef CONFIG_SMP /* PTWs cacheable, inner WB not shareable, outer WB not shareable */ -#define TTB_FLAGS TTB_IRGN_WB|TTB_RGN_OC_WB -#define PMD_FLAGS PMD_SECT_WB -#else +#define TTB_FLAGS_UP TTB_IRGN_WB|TTB_RGN_OC_WB +#define PMD_FLAGS_UP PMD_SECT_WB + /* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */ -#define TTB_FLAGS TTB_IRGN_WBWA|TTB_S|TTB_NOS|TTB_RGN_OC_WBWA -#define PMD_FLAGS PMD_SECT_WBWA|PMD_SECT_S -#endif +#define TTB_FLAGS_SMP TTB_IRGN_WBWA|TTB_S|TTB_NOS|TTB_RGN_OC_WBWA +#define PMD_FLAGS_SMP PMD_SECT_WBWA|PMD_SECT_S ENTRY(cpu_v7_proc_init) mov pc, lr @@ -105,7 +103,8 @@ ENTRY(cpu_v7_switch_mm) #ifdef CONFIG_MMU mov r2, #0 ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id - orr r0, r0, #TTB_FLAGS + ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP) + ALT_UP(orr r0, r0, #TTB_FLAGS_UP) #ifdef CONFIG_ARM_ERRATA_430973 mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB #endif @@ -188,7 +187,8 @@ cpu_v7_name: */ __v7_ca9mp_setup: #ifdef CONFIG_SMP - mrc p15, 0, r0, c1, c0, 1 + ALT_SMP(mrc p15, 0, r0, c1, c0, 1) + ALT_UP(mov r0, #(1 << 6)) @ fake it for UP tst r0, #(1 << 6) @ SMP/nAMP mode enabled? orreq r0, r0, #(1 << 6) | (1 << 0) @ Enable SMP/nAMP mode and mcreq p15, 0, r0, c1, c0, 1 @ TLB ops broadcasting @@ -262,7 +262,8 @@ __v7_setup: #ifdef CONFIG_MMU mcr p15, 0, r10, c8, c7, 0 @ invalidate I + D TLBs mcr p15, 0, r10, c2, c0, 2 @ TTB control register - orr r4, r4, #TTB_FLAGS + ALT_SMP(orr r4, r4, #TTB_FLAGS_SMP) + ALT_UP(orr r4, r4, #TTB_FLAGS_UP) mcr p15, 0, r4, c2, c0, 1 @ load TTB1 mov r10, #0x1f @ domains 0, 1 = manager mcr p15, 0, r10, c3, c0, 0 @ load domain access register @@ -354,10 +355,16 @@ cpu_elf_name: __v7_ca9mp_proc_info: .long 0x410fc090 @ Required ID value .long 0xff0ffff0 @ Mask for ID - .long PMD_TYPE_SECT | \ + ALT_SMP(.long \ + PMD_TYPE_SECT | \ + PMD_SECT_AP_WRITE | \ + PMD_SECT_AP_READ | \ + PMD_FLAGS_SMP) + ALT_UP(.long \ + PMD_TYPE_SECT | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ | \ - PMD_FLAGS + PMD_FLAGS_UP) .long PMD_TYPE_SECT | \ PMD_SECT_XN | \ PMD_SECT_AP_WRITE | \ @@ -380,10 +387,16 @@ __v7_ca9mp_proc_info: __v7_proc_info: .long 0x000f0000 @ Required ID value .long 0x000f0000 @ Mask for ID - .long PMD_TYPE_SECT | \ + ALT_SMP(.long \ + PMD_TYPE_SECT | \ + PMD_SECT_AP_WRITE | \ + PMD_SECT_AP_READ | \ + PMD_FLAGS_SMP) + ALT_UP(.long \ + PMD_TYPE_SECT | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ | \ - PMD_FLAGS + PMD_FLAGS_UP) .long PMD_TYPE_SECT | \ PMD_SECT_XN | \ PMD_SECT_AP_WRITE | \ diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S index f3f288a9546d..53cd5b454673 100644 --- a/arch/arm/mm/tlb-v7.S +++ b/arch/arm/mm/tlb-v7.S @@ -13,6 +13,7 @@ */ #include #include +#include #include #include #include @@ -41,20 +42,15 @@ ENTRY(v7wbi_flush_user_tlb_range) orr r0, r3, r0, lsl #PAGE_SHIFT @ Create initial MVA mov r1, r1, lsl #PAGE_SHIFT 1: -#ifdef CONFIG_SMP - mcr p15, 0, r0, c8, c3, 1 @ TLB invalidate U MVA (shareable) -#else - mcr p15, 0, r0, c8, c7, 1 @ TLB invalidate U MVA -#endif + ALT_SMP(mcr p15, 0, r0, c8, c3, 1) @ TLB invalidate U MVA (shareable) + ALT_UP(mcr p15, 0, r0, c8, c7, 1) @ TLB invalidate U MVA + add r0, r0, #PAGE_SZ cmp r0, r1 blo 1b mov ip, #0 -#ifdef CONFIG_SMP - mcr p15, 0, ip, c7, c1, 6 @ flush BTAC/BTB Inner Shareable -#else - mcr p15, 0, ip, c7, c5, 6 @ flush BTAC/BTB -#endif + ALT_SMP(mcr p15, 0, ip, c7, c1, 6) @ flush BTAC/BTB Inner Shareable + ALT_UP(mcr p15, 0, ip, c7, c5, 6) @ flush BTAC/BTB dsb mov pc, lr ENDPROC(v7wbi_flush_user_tlb_range) @@ -74,20 +70,14 @@ ENTRY(v7wbi_flush_kern_tlb_range) mov r0, r0, lsl #PAGE_SHIFT mov r1, r1, lsl #PAGE_SHIFT 1: -#ifdef CONFIG_SMP - mcr p15, 0, r0, c8, c3, 1 @ TLB invalidate U MVA (shareable) -#else - mcr p15, 0, r0, c8, c7, 1 @ TLB invalidate U MVA -#endif + ALT_SMP(mcr p15, 0, r0, c8, c3, 1) @ TLB invalidate U MVA (shareable) + ALT_UP(mcr p15, 0, r0, c8, c7, 1) @ TLB invalidate U MVA add r0, r0, #PAGE_SZ cmp r0, r1 blo 1b mov r2, #0 -#ifdef CONFIG_SMP - mcr p15, 0, r2, c7, c1, 6 @ flush BTAC/BTB Inner Shareable -#else - mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB -#endif + ALT_SMP(mcr p15, 0, r2, c7, c1, 6) @ flush BTAC/BTB Inner Shareable + ALT_UP(mcr p15, 0, r2, c7, c5, 6) @ flush BTAC/BTB dsb isb mov pc, lr @@ -99,5 +89,6 @@ ENDPROC(v7wbi_flush_kern_tlb_range) ENTRY(v7wbi_tlb_fns) .long v7wbi_flush_user_tlb_range .long v7wbi_flush_kern_tlb_range - .long v7wbi_tlb_flags + ALT_SMP(.long v7wbi_tlb_flags_smp) + ALT_UP(.long v7wbi_tlb_flags_up) .size v7wbi_tlb_fns, . - v7wbi_tlb_fns -- cgit v1.2.3 From 81d11955bf0b5ae25e3adbec930cd84840385dae Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Tue, 21 Sep 2010 17:16:40 +0100 Subject: ARM: 6405/1: Handle __flush_icache_all for CONFIG_SMP_ON_UP Do this by adding flush_icache_all to cache_fns for ARMv6 and 7. As flush_icache_all may neeed to be called from flush_kern_cache_all, add it as the first entry in the cache_fns. Note that now we can remove the ARM_ERRATA_411920 dependency to !SMP so it can be selected on UP ARMv6 processors, such as omap2. Signed-off-by: Tony Lindgren Signed-off-by: Anand Gadiyar Signed-off-by: Russell King --- arch/arm/mm/cache-v6.S | 30 +++++++++++++++++++----------- arch/arm/mm/cache-v7.S | 16 ++++++++++++++++ 2 files changed, 35 insertions(+), 11 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S index 86aa689ef1aa..99fa688dfadd 100644 --- a/arch/arm/mm/cache-v6.S +++ b/arch/arm/mm/cache-v6.S @@ -21,18 +21,22 @@ #define D_CACHE_LINE_SIZE 32 #define BTB_FLUSH_SIZE 8 -#ifdef CONFIG_ARM_ERRATA_411920 /* - * Invalidate the entire I cache (this code is a workaround for the ARM1136 - * erratum 411920 - Invalidate Instruction Cache operation can fail. This - * erratum is present in 1136, 1156 and 1176. It does not affect the MPCore. + * v6_flush_icache_all() + * + * Flush the whole I-cache. * - * Registers: - * r0 - set to 0 - * r1 - corrupted + * ARM1136 erratum 411920 - Invalidate Instruction Cache operation can fail. + * This erratum is present in 1136, 1156 and 1176. It does not affect the + * MPCore. + * + * Registers: + * r0 - set to 0 + * r1 - corrupted */ -ENTRY(v6_icache_inval_all) +ENTRY(v6_flush_icache_all) mov r0, #0 +#ifdef CONFIG_ARM_ERRATA_411920 mrs r1, cpsr cpsid ifa @ disable interrupts mcr p15, 0, r0, c7, c5, 0 @ invalidate entire I-cache @@ -43,8 +47,11 @@ ENTRY(v6_icache_inval_all) .rept 11 @ ARM Ltd recommends at least nop @ 11 NOPs .endr - mov pc, lr +#else + mcr p15, 0, r0, c7, c5, 0 @ invalidate I-cache #endif + mov pc, lr +ENDPROC(v6_flush_icache_all) /* * v6_flush_cache_all() @@ -60,7 +67,7 @@ ENTRY(v6_flush_kern_cache_all) #ifndef CONFIG_ARM_ERRATA_411920 mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate #else - b v6_icache_inval_all + b v6_flush_icache_all #endif #else mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate @@ -138,7 +145,7 @@ ENTRY(v6_coherent_user_range) #ifndef CONFIG_ARM_ERRATA_411920 mcr p15, 0, r0, c7, c5, 0 @ I+BTB cache invalidate #else - b v6_icache_inval_all + b v6_flush_icache_all #endif #else mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB @@ -312,6 +319,7 @@ ENDPROC(v6_dma_unmap_area) .type v6_cache_fns, #object ENTRY(v6_cache_fns) + .long v6_flush_icache_all .long v6_flush_kern_cache_all .long v6_flush_user_cache_all .long v6_flush_user_cache_range diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index e8ea1a071f43..a3ebf7a4f49b 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -17,6 +17,21 @@ #include "proc-macros.S" +/* + * v7_flush_icache_all() + * + * Flush the whole I-cache. + * + * Registers: + * r0 - set to 0 + */ +ENTRY(v7_flush_icache_all) + mov r0, #0 + ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable + ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate + mov pc, lr +ENDPROC(v7_flush_icache_all) + /* * v7_flush_dcache_all() * @@ -303,6 +318,7 @@ ENDPROC(v7_dma_unmap_area) .type v7_cache_fns, #object ENTRY(v7_cache_fns) + .long v7_flush_icache_all .long v7_flush_kern_cache_all .long v7_flush_user_cache_all .long v7_flush_user_cache_range -- cgit v1.2.3 From c4e259c859538e94007d1f04a488540375189551 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Mon, 13 Sep 2010 16:19:41 +0100 Subject: ARM: 6386/1: flush_ptrace_access: invalidate correct I-cache alias copy_to_user_page can be used by access_process_vm to write to an executable page of a process using a mapping acquired by kmap. For systems with I-cache aliasing, flushing the I-cache using the Kernel mapping may leave stale data in the I-cache if the user mapping is of a different colour. This patch introduces a flush_icache_alias function to flush.c, which calls flush_icache_range with a mapping of the specified colour. flush_ptrace_access is then modified to call this new function instead of coherent_kern_range in the case of an aliasing I-cache and a non-aliasing D-cache. Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/mm/flush.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 2332b774c6b9..391ffae75098 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -40,6 +40,18 @@ static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr) : "cc"); } +static void flush_icache_alias(unsigned long pfn, unsigned long vaddr, unsigned long len) +{ + unsigned long colour = CACHE_COLOUR(vaddr); + unsigned long offset = vaddr & (PAGE_SIZE - 1); + unsigned long to; + + set_pte_ext(TOP_PTE(ALIAS_FLUSH_START) + colour, pfn_pte(pfn, PAGE_KERNEL), 0); + to = ALIAS_FLUSH_START + (colour << PAGE_SHIFT) + offset; + flush_tlb_kernel_page(to); + flush_icache_range(to, to + len); +} + void flush_cache_mm(struct mm_struct *mm) { if (cache_is_vivt()) { @@ -90,8 +102,10 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged()) __flush_icache_all(); } + #else -#define flush_pfn_alias(pfn,vaddr) do { } while (0) +#define flush_pfn_alias(pfn,vaddr) do { } while (0) +#define flush_icache_alias(pfn,vaddr,len) do { } while (0) #endif static void flush_ptrace_access_other(void *args) @@ -117,10 +131,13 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, return; } - /* VIPT non-aliasing cache */ + /* VIPT non-aliasing D-cache */ if (vma->vm_flags & VM_EXEC) { unsigned long addr = (unsigned long)kaddr; - __cpuc_coherent_kern_range(addr, addr + len); + if (icache_is_vipt_aliasing()) + flush_icache_alias(page_to_pfn(page), uaddr, len); + else + __cpuc_coherent_kern_range(addr, addr + len); if (cache_ops_need_broadcast()) smp_call_function(flush_ptrace_access_other, NULL, 1); -- cgit v1.2.3 From 842eab40b6920819fff5caeefdb709f07d3f8e23 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 1 Oct 2010 14:12:22 +0100 Subject: ARM: vmlinux.lds: Refer to start of .data using _sdata rather than _data Use _sdata as the start of the data section, rather than _data. Signed-off-by: Russell King --- arch/arm/mm/init.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 7185b00650fe..36c4553ffcce 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -277,7 +277,7 @@ void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc) /* Register the kernel text, kernel data and initrd with memblock. */ #ifdef CONFIG_XIP_KERNEL - memblock_reserve(__pa(_data), _end - _data); + memblock_reserve(__pa(_sdata), _end - _sdata); #else memblock_reserve(__pa(_stext), _end - _stext); #endif @@ -545,7 +545,7 @@ void __init mem_init(void) MLK_ROUNDUP(__init_begin, __init_end), MLK_ROUNDUP(_text, _etext), - MLK_ROUNDUP(_data, _edata)); + MLK_ROUNDUP(_sdata, _edata)); #undef MLK #undef MLM -- cgit v1.2.3