diff options
Diffstat (limited to 'arch')
30 files changed, 331 insertions, 220 deletions
diff --git a/arch/arm/mach-s5pv310/Kconfig b/arch/arm/mach-s5pv310/Kconfig index 09c4c21b70cc..b2a9acc5185f 100644 --- a/arch/arm/mach-s5pv310/Kconfig +++ b/arch/arm/mach-s5pv310/Kconfig @@ -122,6 +122,7 @@ config MACH_SMDKV310 select S3C_DEV_HSMMC2 select S3C_DEV_HSMMC3 select S5PV310_DEV_PD + select S5PV310_DEV_SYSMMU select S5PV310_SETUP_I2C1 select S5PV310_SETUP_SDHCI help diff --git a/arch/arm/mach-s5pv310/include/mach/map.h b/arch/arm/mach-s5pv310/include/mach/map.h index 74d400625a23..3060f78e12ab 100644 --- a/arch/arm/mach-s5pv310/include/mach/map.h +++ b/arch/arm/mach-s5pv310/include/mach/map.h @@ -124,8 +124,6 @@ #define S5PV310_PA_SYSMMU_TV 0x12E20000 #define S5PV310_PA_SYSMMU_MFC_L 0x13620000 #define S5PV310_PA_SYSMMU_MFC_R 0x13630000 -#define S5PV310_SYSMMU_TOTAL_IPNUM 16 -#define S5P_SYSMMU_TOTAL_IPNUM S5PV310_SYSMMU_TOTAL_IPNUM /* compatibiltiy defines. */ #define S3C_PA_UART S5PV310_PA_UART diff --git a/arch/arm/mach-s5pv310/include/mach/sysmmu.h b/arch/arm/mach-s5pv310/include/mach/sysmmu.h index 662fe85ff4d5..598fc5c9211b 100644 --- a/arch/arm/mach-s5pv310/include/mach/sysmmu.h +++ b/arch/arm/mach-s5pv310/include/mach/sysmmu.h @@ -13,6 +13,9 @@ #ifndef __ASM_ARM_ARCH_SYSMMU_H #define __ASM_ARM_ARCH_SYSMMU_H __FILE__ +#define S5PV310_SYSMMU_TOTAL_IPNUM 16 +#define S5P_SYSMMU_TOTAL_IPNUM S5PV310_SYSMMU_TOTAL_IPNUM + enum s5pv310_sysmmu_ips { SYSMMU_MDMA, SYSMMU_SSS, @@ -32,7 +35,7 @@ enum s5pv310_sysmmu_ips { SYSMMU_MFC_R, }; -static char *sysmmu_ips_name[S5P_SYSMMU_TOTAL_IPNUM] = { +static char *sysmmu_ips_name[S5PV310_SYSMMU_TOTAL_IPNUM] = { "SYSMMU_MDMA" , "SYSMMU_SSS" , "SYSMMU_FIMC0" , diff --git a/arch/arm/plat-s5p/Kconfig b/arch/arm/plat-s5p/Kconfig index deb39951a22e..557f8c507f6d 100644 --- a/arch/arm/plat-s5p/Kconfig +++ b/arch/arm/plat-s5p/Kconfig @@ -37,6 +37,14 @@ config S5P_GPIO_INT help Common code for the GPIO interrupts (other than external interrupts.) +comment "System MMU" + +config S5P_SYSTEM_MMU + bool "S5P SYSTEM MMU" + depends on ARCH_S5PV310 + help + Say Y here if you want to enable System MMU + config S5P_DEV_FIMC0 bool help @@ -66,19 +74,3 @@ config S5P_DEV_CSIS1 bool help Compile in platform device definitions for MIPI-CSIS channel 1 - -menuconfig S5P_SYSMMU - bool "SYSMMU support" - depends on ARCH_S5PV310 - help - This is a System MMU driver for Samsung ARM based Soc. - -if S5P_SYSMMU - -config S5P_SYSMMU_DEBUG - bool "Enables debug messages" - depends on S5P_SYSMMU - help - This enables SYSMMU driver debug massages. - -endif diff --git a/arch/arm/plat-s5p/Makefile b/arch/arm/plat-s5p/Makefile index 92efe1adcfd6..4bd5cf908977 100644 --- a/arch/arm/plat-s5p/Makefile +++ b/arch/arm/plat-s5p/Makefile @@ -19,6 +19,7 @@ obj-y += clock.o obj-y += irq.o obj-$(CONFIG_S5P_EXT_INT) += irq-eint.o obj-$(CONFIG_S5P_GPIO_INT) += irq-gpioint.o +obj-$(CONFIG_S5P_SYSTEM_MMU) += sysmmu.o obj-$(CONFIG_PM) += pm.o obj-$(CONFIG_PM) += irq-pm.o @@ -30,4 +31,3 @@ obj-$(CONFIG_S5P_DEV_FIMC2) += dev-fimc2.o obj-$(CONFIG_S5P_DEV_ONENAND) += dev-onenand.o obj-$(CONFIG_S5P_DEV_CSIS0) += dev-csis0.o obj-$(CONFIG_S5P_DEV_CSIS1) += dev-csis1.o -obj-$(CONFIG_S5P_SYSMMU) += sysmmu.o diff --git a/arch/arm/plat-s5p/include/plat/sysmmu.h b/arch/arm/plat-s5p/include/plat/sysmmu.h deleted file mode 100644 index db298fc5438a..000000000000 --- a/arch/arm/plat-s5p/include/plat/sysmmu.h +++ /dev/null @@ -1,23 +0,0 @@ -/* linux/arch/arm/plat-s5p/include/plat/sysmmu.h - * - * Copyright (c) 2010 Samsung Electronics Co., Ltd. - * http://www.samsung.com/ - * - * Samsung sysmmu driver - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#ifndef __ASM_PLAT_S5P_SYSMMU_H -#define __ASM_PLAT_S5P_SYSMMU_H __FILE__ - -/* debug macro */ -#ifdef CONFIG_S5P_SYSMMU_DEBUG -#define sysmmu_debug(fmt, arg...) printk(KERN_INFO "[%s] " fmt, __func__, ## arg) -#else -#define sysmmu_debug(fmt, arg...) do { } while (0) -#endif - -#endif /* __ASM_PLAT_S5P_SYSMMU_H */ diff --git a/arch/arm/plat-s5p/sysmmu.c b/arch/arm/plat-s5p/sysmmu.c index d804914dc2e2..ffe8a48bc3c1 100644 --- a/arch/arm/plat-s5p/sysmmu.c +++ b/arch/arm/plat-s5p/sysmmu.c @@ -16,8 +16,6 @@ #include <mach/regs-sysmmu.h> #include <mach/sysmmu.h> -#include <plat/sysmmu.h> - struct sysmmu_controller s5p_sysmmu_cntlrs[S5P_SYSMMU_TOTAL_IPNUM]; void s5p_sysmmu_register(struct sysmmu_controller *sysmmuconp) @@ -123,7 +121,7 @@ static int s5p_sysmmu_set_tablebase(sysmmu_ips ips) : "=r" (pg) : : "cc"); \ pg &= ~0x3fff; - sysmmu_debug("CP15 TTBR0 : 0x%x\n", pg); + printk(KERN_INFO "%s: CP15 TTBR0 : 0x%x\n", __func__, pg); /* Set sysmmu page table base address */ __raw_writel(pg, sysmmuconp->regs + S5P_PT_BASE_ADDR); diff --git a/arch/arm/plat-samsung/include/plat/pm.h b/arch/arm/plat-samsung/include/plat/pm.h index d9025e377675..30518cc9a67c 100644 --- a/arch/arm/plat-samsung/include/plat/pm.h +++ b/arch/arm/plat-samsung/include/plat/pm.h @@ -17,6 +17,8 @@ #include <linux/irq.h> +struct sys_device; + #ifdef CONFIG_PM extern __init int s3c_pm_init(void); diff --git a/arch/blackfin/include/asm/bfin_serial.h b/arch/blackfin/include/asm/bfin_serial.h index 1ff9f1468c02..7dbc664eab1e 100644 --- a/arch/blackfin/include/asm/bfin_serial.h +++ b/arch/blackfin/include/asm/bfin_serial.h @@ -10,6 +10,7 @@ #define __BFIN_ASM_SERIAL_H__ #include <linux/serial_core.h> +#include <linux/spinlock.h> #include <mach/anomaly.h> #include <mach/bfin_serial.h> @@ -41,6 +42,7 @@ struct bfin_serial_port { struct circ_buf rx_dma_buf; struct timer_list rx_dma_timer; int rx_dma_nrows; + spinlock_t rx_lock; unsigned int tx_dma_channel; unsigned int rx_dma_channel; struct work_struct tx_dma_workqueue; diff --git a/arch/microblaze/include/asm/irqflags.h b/arch/microblaze/include/asm/irqflags.h index 5fd31905775d..c4532f032b3b 100644 --- a/arch/microblaze/include/asm/irqflags.h +++ b/arch/microblaze/include/asm/irqflags.h @@ -12,7 +12,7 @@ #include <linux/types.h> #include <asm/registers.h> -#ifdef CONFIG_XILINX_MICROBLAZE0_USE_MSR_INSTR +#if CONFIG_XILINX_MICROBLAZE0_USE_MSR_INSTR static inline unsigned long arch_local_irq_save(void) { diff --git a/arch/microblaze/include/asm/pgtable.h b/arch/microblaze/include/asm/pgtable.h index b23f68075879..885574a73f01 100644 --- a/arch/microblaze/include/asm/pgtable.h +++ b/arch/microblaze/include/asm/pgtable.h @@ -411,20 +411,19 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) static inline unsigned long pte_update(pte_t *p, unsigned long clr, unsigned long set) { - unsigned long old, tmp, msr; - - __asm__ __volatile__("\ - msrclr %2, 0x2\n\ - nop\n\ - lw %0, %4, r0\n\ - andn %1, %0, %5\n\ - or %1, %1, %6\n\ - sw %1, %4, r0\n\ - mts rmsr, %2\n\ - nop" - : "=&r" (old), "=&r" (tmp), "=&r" (msr), "=m" (*p) - : "r" ((unsigned long)(p + 1) - 4), "r" (clr), "r" (set), "m" (*p) - : "cc"); + unsigned long flags, old, tmp; + + raw_local_irq_save(flags); + + __asm__ __volatile__( "lw %0, %2, r0 \n" + "andn %1, %0, %3 \n" + "or %1, %1, %4 \n" + "sw %1, %2, r0 \n" + : "=&r" (old), "=&r" (tmp) + : "r" ((unsigned long)(p + 1) - 4), "r" (clr), "r" (set) + : "cc"); + + raw_local_irq_restore(flags); return old; } diff --git a/arch/microblaze/kernel/cpu/pvr.c b/arch/microblaze/kernel/cpu/pvr.c index e01afa68273e..488c1ed24e38 100644 --- a/arch/microblaze/kernel/cpu/pvr.c +++ b/arch/microblaze/kernel/cpu/pvr.c @@ -27,7 +27,7 @@ register unsigned tmp __asm__("r3"); \ tmp = 0x0; /* Prevent warning about unused */ \ __asm__ __volatile__ ( \ - "mfs %0, rpvr" #pvrid ";" \ + "mfs %0, rpvr" #pvrid ";" \ : "=r" (tmp) : : "memory"); \ val = tmp; \ } @@ -54,7 +54,7 @@ int cpu_has_pvr(void) if (!(flags & PVR_MSR_BIT)) return 0; - get_single_pvr(0x00, pvr0); + get_single_pvr(0, pvr0); pr_debug("%s: pvr0 is 0x%08x\n", __func__, pvr0); if (pvr0 & PVR0_PVR_FULL_MASK) diff --git a/arch/microblaze/kernel/head.S b/arch/microblaze/kernel/head.S index 0db20b5abb54..778a5ce2e4fc 100644 --- a/arch/microblaze/kernel/head.S +++ b/arch/microblaze/kernel/head.S @@ -62,15 +62,14 @@ real_start: andi r1, r1, ~2 mts rmsr, r1 /* - * Here is checking mechanism which check if Microblaze has msr instructions - * We load msr and compare it with previous r1 value - if is the same, - * msr instructions works if not - cpu don't have them. + * According to Xilinx, msrclr instruction behaves like 'mfs rX,rpc' + * if the msrclr instruction is not enabled. We use this to detect + * if the opcode is available, by issuing msrclr and then testing the result. + * r8 == 0 - msr instructions are implemented + * r8 != 0 - msr instructions are not implemented */ - /* r8=0 - I have msr instr, 1 - I don't have them */ - rsubi r0, r0, 1 /* set the carry bit */ - msrclr r0, 0x4 /* try to clear it */ - /* read the carry bit, r8 will be '0' if msrclr exists */ - addik r8, r0, 0 + msrclr r8, 0 /* clear nothing - just read msr for test */ + cmpu r8, r8, r1 /* r1 must contain msr reg content */ /* r7 may point to an FDT, or there may be one linked in. if it's in r7, we've got to save it away ASAP. diff --git a/arch/microblaze/kernel/setup.c b/arch/microblaze/kernel/setup.c index bb1558e4b283..9312fbb37efd 100644 --- a/arch/microblaze/kernel/setup.c +++ b/arch/microblaze/kernel/setup.c @@ -161,11 +161,11 @@ void __init machine_early_init(const char *cmdline, unsigned int ram, #if CONFIG_XILINX_MICROBLAZE0_USE_MSR_INSTR if (msr) eprintk("!!!Your kernel has setup MSR instruction but " - "CPU don't have it %d\n", msr); + "CPU don't have it %x\n", msr); #else if (!msr) eprintk("!!!Your kernel not setup MSR instruction but " - "CPU have it %d\n", msr); + "CPU have it %x\n", msr); #endif for (src = __ivt_start; src < __ivt_end; src++, dst++) diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h index 57650ab4a5f5..1cd6d26a0a8d 100644 --- a/arch/x86/include/asm/entry_arch.h +++ b/arch/x86/include/asm/entry_arch.h @@ -16,10 +16,13 @@ BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR) BUILD_INTERRUPT(irq_move_cleanup_interrupt,IRQ_MOVE_CLEANUP_VECTOR) BUILD_INTERRUPT(reboot_interrupt,REBOOT_VECTOR) -.irpc idx, "01234567" +.irp idx,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \ + 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +.if NUM_INVALIDATE_TLB_VECTORS > \idx BUILD_INTERRUPT3(invalidate_interrupt\idx, (INVALIDATE_TLB_VECTOR_START)+\idx, smp_invalidate_interrupt) +.endif .endr #endif diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h index 0274ec5a7e62..bb9efe8706e2 100644 --- a/arch/x86/include/asm/hw_irq.h +++ b/arch/x86/include/asm/hw_irq.h @@ -45,6 +45,30 @@ extern void invalidate_interrupt4(void); extern void invalidate_interrupt5(void); extern void invalidate_interrupt6(void); extern void invalidate_interrupt7(void); +extern void invalidate_interrupt8(void); +extern void invalidate_interrupt9(void); +extern void invalidate_interrupt10(void); +extern void invalidate_interrupt11(void); +extern void invalidate_interrupt12(void); +extern void invalidate_interrupt13(void); +extern void invalidate_interrupt14(void); +extern void invalidate_interrupt15(void); +extern void invalidate_interrupt16(void); +extern void invalidate_interrupt17(void); +extern void invalidate_interrupt18(void); +extern void invalidate_interrupt19(void); +extern void invalidate_interrupt20(void); +extern void invalidate_interrupt21(void); +extern void invalidate_interrupt22(void); +extern void invalidate_interrupt23(void); +extern void invalidate_interrupt24(void); +extern void invalidate_interrupt25(void); +extern void invalidate_interrupt26(void); +extern void invalidate_interrupt27(void); +extern void invalidate_interrupt28(void); +extern void invalidate_interrupt29(void); +extern void invalidate_interrupt30(void); +extern void invalidate_interrupt31(void); extern void irq_move_cleanup_interrupt(void); extern void reboot_interrupt(void); diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h index 6af0894dafb4..4980f48bbbb7 100644 --- a/arch/x86/include/asm/irq_vectors.h +++ b/arch/x86/include/asm/irq_vectors.h @@ -1,6 +1,7 @@ #ifndef _ASM_X86_IRQ_VECTORS_H #define _ASM_X86_IRQ_VECTORS_H +#include <linux/threads.h> /* * Linux IRQ vector layout. * @@ -16,8 +17,8 @@ * Vectors 0 ... 31 : system traps and exceptions - hardcoded events * Vectors 32 ... 127 : device interrupts * Vector 128 : legacy int80 syscall interface - * Vectors 129 ... 237 : device interrupts - * Vectors 238 ... 255 : special interrupts + * Vectors 129 ... INVALIDATE_TLB_VECTOR_START-1 : device interrupts + * Vectors INVALIDATE_TLB_VECTOR_START ... 255 : special interrupts * * 64-bit x86 has per CPU IDT tables, 32-bit has one shared IDT table. * @@ -96,37 +97,43 @@ #define THRESHOLD_APIC_VECTOR 0xf9 #define REBOOT_VECTOR 0xf8 -/* f0-f7 used for spreading out TLB flushes: */ -#define INVALIDATE_TLB_VECTOR_END 0xf7 -#define INVALIDATE_TLB_VECTOR_START 0xf0 -#define NUM_INVALIDATE_TLB_VECTORS 8 - -/* - * Local APIC timer IRQ vector is on a different priority level, - * to work around the 'lost local interrupt if more than 2 IRQ - * sources per level' errata. - */ -#define LOCAL_TIMER_VECTOR 0xef - /* * Generic system vector for platform specific use */ -#define X86_PLATFORM_IPI_VECTOR 0xed +#define X86_PLATFORM_IPI_VECTOR 0xf7 /* * IRQ work vector: */ -#define IRQ_WORK_VECTOR 0xec +#define IRQ_WORK_VECTOR 0xf6 -#define UV_BAU_MESSAGE 0xea +#define UV_BAU_MESSAGE 0xf5 /* * Self IPI vector for machine checks */ -#define MCE_SELF_VECTOR 0xeb +#define MCE_SELF_VECTOR 0xf4 /* Xen vector callback to receive events in a HVM domain */ -#define XEN_HVM_EVTCHN_CALLBACK 0xe9 +#define XEN_HVM_EVTCHN_CALLBACK 0xf3 + +/* + * Local APIC timer IRQ vector is on a different priority level, + * to work around the 'lost local interrupt if more than 2 IRQ + * sources per level' errata. + */ +#define LOCAL_TIMER_VECTOR 0xef + +/* up to 32 vectors used for spreading out TLB flushes: */ +#if NR_CPUS <= 32 +# define NUM_INVALIDATE_TLB_VECTORS NR_CPUS +#else +# define NUM_INVALIDATE_TLB_VECTORS 32 +#endif + +#define INVALIDATE_TLB_VECTOR_END 0xee +#define INVALIDATE_TLB_VECTOR_START \ + (INVALIDATE_TLB_VECTOR_END - NUM_INVALIDATE_TLB_VECTORS + 1) #define NR_VECTORS 256 diff --git a/arch/x86/include/asm/page_types.h b/arch/x86/include/asm/page_types.h index 1df66211fd1b..731d211a1b20 100644 --- a/arch/x86/include/asm/page_types.h +++ b/arch/x86/include/asm/page_types.h @@ -2,6 +2,7 @@ #define _ASM_X86_PAGE_DEFS_H #include <linux/const.h> +#include <linux/types.h> /* PAGE_SHIFT determines the page size */ #define PAGE_SHIFT 12 @@ -45,9 +46,16 @@ extern int devmem_is_allowed(unsigned long pagenr); extern unsigned long max_low_pfn_mapped; extern unsigned long max_pfn_mapped; +static inline phys_addr_t get_max_mapped(void) +{ + return (phys_addr_t)max_pfn_mapped << PAGE_SHIFT; +} + extern unsigned long init_memory_mapping(unsigned long start, unsigned long end); +void init_memory_mapping_high(void); + extern void initmem_init(unsigned long start_pfn, unsigned long end_pfn, int acpi, int k8); extern void free_initmem(void); diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c index 5955a7800a96..7b1e8e10b89c 100644 --- a/arch/x86/kernel/aperture_64.c +++ b/arch/x86/kernel/aperture_64.c @@ -13,7 +13,7 @@ #include <linux/kernel.h> #include <linux/types.h> #include <linux/init.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/mmzone.h> #include <linux/pci_ids.h> #include <linux/pci.h> @@ -57,7 +57,7 @@ static void __init insert_aperture_resource(u32 aper_base, u32 aper_size) static u32 __init allocate_aperture(void) { u32 aper_size; - void *p; + unsigned long addr; /* aper_size should <= 1G */ if (fallback_aper_order > 5) @@ -83,27 +83,26 @@ static u32 __init allocate_aperture(void) * so don't use 512M below as gart iommu, leave the space for kernel * code for safe */ - p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20); + addr = memblock_find_in_range(0, 1ULL<<32, aper_size, 512ULL<<20); + if (addr == MEMBLOCK_ERROR || addr + aper_size > 0xffffffff) { + printk(KERN_ERR + "Cannot allocate aperture memory hole (%lx,%uK)\n", + addr, aper_size>>10); + return 0; + } + memblock_x86_reserve_range(addr, addr + aper_size, "aperture64"); /* * Kmemleak should not scan this block as it may not be mapped via the * kernel direct mapping. */ - kmemleak_ignore(p); - if (!p || __pa(p)+aper_size > 0xffffffff) { - printk(KERN_ERR - "Cannot allocate aperture memory hole (%p,%uK)\n", - p, aper_size>>10); - if (p) - free_bootmem(__pa(p), aper_size); - return 0; - } + kmemleak_ignore(phys_to_virt(addr)); printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n", - aper_size >> 10, __pa(p)); - insert_aperture_resource((u32)__pa(p), aper_size); - register_nosave_region((u32)__pa(p) >> PAGE_SHIFT, - (u32)__pa(p+aper_size) >> PAGE_SHIFT); + aper_size >> 10, addr); + insert_aperture_resource((u32)addr, aper_size); + register_nosave_region(addr >> PAGE_SHIFT, + (addr+aper_size) >> PAGE_SHIFT); - return (u32)__pa(p); + return (u32)addr; } diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index 294f26da0c0c..0b5e2b546566 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c @@ -847,15 +847,21 @@ static int __init parse_memopt(char *p) if (!p) return -EINVAL; -#ifdef CONFIG_X86_32 if (!strcmp(p, "nopentium")) { +#ifdef CONFIG_X86_32 setup_clear_cpu_cap(X86_FEATURE_PSE); return 0; - } +#else + printk(KERN_WARNING "mem=nopentium ignored! (only supported on x86_32)\n"); + return -EINVAL; #endif + } userdef = 1; mem_size = memparse(p, &p); + /* don't remove all of memory when handling "mem={invalid}" param */ + if (mem_size == 0) + return -EINVAL; e820_remove_range(mem_size, ULLONG_MAX - mem_size, E820_RAM, 1); return 0; diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index aed1ffbeb0c9..891268c645ea 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -975,9 +975,12 @@ apicinterrupt X86_PLATFORM_IPI_VECTOR \ x86_platform_ipi smp_x86_platform_ipi #ifdef CONFIG_SMP -.irpc idx, "01234567" +.irp idx,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \ + 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +.if NUM_INVALIDATE_TLB_VECTORS > \idx apicinterrupt (INVALIDATE_TLB_VECTOR_START)+\idx \ invalidate_interrupt\idx smp_invalidate_interrupt +.endif .endr #endif diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c index c752e973958d..7aad10a63e04 100644 --- a/arch/x86/kernel/irqinit.c +++ b/arch/x86/kernel/irqinit.c @@ -164,14 +164,77 @@ static void __init smp_intr_init(void) alloc_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt); /* IPIs for invalidation */ - alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+0, invalidate_interrupt0); - alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+1, invalidate_interrupt1); - alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+2, invalidate_interrupt2); - alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+3, invalidate_interrupt3); - alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+4, invalidate_interrupt4); - alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+5, invalidate_interrupt5); - alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+6, invalidate_interrupt6); - alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+7, invalidate_interrupt7); +#define ALLOC_INVTLB_VEC(NR) \ + alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+NR, \ + invalidate_interrupt##NR) + + switch (NUM_INVALIDATE_TLB_VECTORS) { + default: + ALLOC_INVTLB_VEC(31); + case 31: + ALLOC_INVTLB_VEC(30); + case 30: + ALLOC_INVTLB_VEC(29); + case 29: + ALLOC_INVTLB_VEC(28); + case 28: + ALLOC_INVTLB_VEC(27); + case 27: + ALLOC_INVTLB_VEC(26); + case 26: + ALLOC_INVTLB_VEC(25); + case 25: + ALLOC_INVTLB_VEC(24); + case 24: + ALLOC_INVTLB_VEC(23); + case 23: + ALLOC_INVTLB_VEC(22); + case 22: + ALLOC_INVTLB_VEC(21); + case 21: + ALLOC_INVTLB_VEC(20); + case 20: + ALLOC_INVTLB_VEC(19); + case 19: + ALLOC_INVTLB_VEC(18); + case 18: + ALLOC_INVTLB_VEC(17); + case 17: + ALLOC_INVTLB_VEC(16); + case 16: + ALLOC_INVTLB_VEC(15); + case 15: + ALLOC_INVTLB_VEC(14); + case 14: + ALLOC_INVTLB_VEC(13); + case 13: + ALLOC_INVTLB_VEC(12); + case 12: + ALLOC_INVTLB_VEC(11); + case 11: + ALLOC_INVTLB_VEC(10); + case 10: + ALLOC_INVTLB_VEC(9); + case 9: + ALLOC_INVTLB_VEC(8); + case 8: + ALLOC_INVTLB_VEC(7); + case 7: + ALLOC_INVTLB_VEC(6); + case 6: + ALLOC_INVTLB_VEC(5); + case 5: + ALLOC_INVTLB_VEC(4); + case 4: + ALLOC_INVTLB_VEC(3); + case 3: + ALLOC_INVTLB_VEC(2); + case 2: + ALLOC_INVTLB_VEC(1); + case 1: + ALLOC_INVTLB_VEC(0); + break; + } /* IPI for generic function call */ alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt); diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 12023412fdf7..ac909ba297b9 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -293,10 +293,32 @@ static void __init init_gbpages(void) else direct_gbpages = 0; } + +static void __init cleanup_highmap_brk_end(void) +{ + pud_t *pud; + pmd_t *pmd; + + mmu_cr4_features = read_cr4(); + + /* + * _brk_end cannot change anymore, but it and _end may be + * located on different 2M pages. cleanup_highmap(), however, + * can only consider _end when it runs, so destroy any + * mappings beyond _brk_end here. + */ + pud = pud_offset(pgd_offset_k(_brk_end), _brk_end); + pmd = pmd_offset(pud, _brk_end - 1); + while (++pmd <= pmd_offset(pud, (unsigned long)_end - 1)) + pmd_clear(pmd); +} #else static inline void init_gbpages(void) { } +static inline void cleanup_highmap_brk_end(void) +{ +} #endif static void __init reserve_brk(void) @@ -307,6 +329,8 @@ static void __init reserve_brk(void) /* Mark brk area as locked down and no longer taking any new allocations */ _brk_start = 0; + + cleanup_highmap_brk_end(); } #ifdef CONFIG_BLK_DEV_INITRD @@ -680,15 +704,6 @@ static int __init parse_reservelow(char *p) early_param("reservelow", parse_reservelow); -static u64 __init get_max_mapped(void) -{ - u64 end = max_pfn_mapped; - - end <<= PAGE_SHIFT; - - return end; -} - /* * Determine if we were loaded by an EFI loader. If so, then we have also been * passed the efi memmap, systab, etc., so we should use these data structures @@ -950,14 +965,6 @@ void __init setup_arch(char **cmdline_p) max_low_pfn_mapped = init_memory_mapping(0, max_low_pfn<<PAGE_SHIFT); max_pfn_mapped = max_low_pfn_mapped; -#ifdef CONFIG_X86_64 - if (max_pfn > max_low_pfn) { - max_pfn_mapped = init_memory_mapping(1UL<<32, - max_pfn<<PAGE_SHIFT); - /* can we preseve max_low_pfn ?*/ - max_low_pfn = max_pfn; - } -#endif memblock.current_limit = get_max_mapped(); /* diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 25bd1bc5aad2..54ce246a383e 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -1150,8 +1150,8 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) kvm_load_ldt(svm->host.ldt); #ifdef CONFIG_X86_64 loadsegment(fs, svm->host.fs); - load_gs_index(svm->host.gs); wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs); + load_gs_index(svm->host.gs); #else loadsegment(gs, svm->host.gs); #endif diff --git a/arch/x86/mm/amdtopology_64.c b/arch/x86/mm/amdtopology_64.c index c7fae38c4080..2523c3554de5 100644 --- a/arch/x86/mm/amdtopology_64.c +++ b/arch/x86/mm/amdtopology_64.c @@ -278,12 +278,14 @@ int __init amd_scan_nodes(void) apicid_base = boot_cpu_physical_apicid; } - for_each_node_mask(i, node_possible_map) { - int j; - + for_each_node_mask(i, node_possible_map) memblock_x86_register_active_regions(i, nodes[i].start >> PAGE_SHIFT, nodes[i].end >> PAGE_SHIFT); + init_memory_mapping_high(); + for_each_node_mask(i, node_possible_map) { + int j; + for (j = apicid_base; j < cores + apicid_base; j++) set_apicid_to_node((i << bits) + j, i); setup_node_bootmem(i, nodes[i].start, nodes[i].end); diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index 947f42abe820..b8054e087ead 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -33,7 +33,7 @@ int direct_gbpages static void __init find_early_table_space(unsigned long end, int use_pse, int use_gbpages) { - unsigned long puds, pmds, ptes, tables, start; + unsigned long puds, pmds, ptes, tables, start = 0, good_end = end; phys_addr_t base; puds = (end + PUD_SIZE - 1) >> PUD_SHIFT; @@ -65,20 +65,11 @@ static void __init find_early_table_space(unsigned long end, int use_pse, #ifdef CONFIG_X86_32 /* for fixmap */ tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE); -#endif - /* - * RED-PEN putting page tables only on node 0 could - * cause a hotspot and fill up ZONE_DMA. The page tables - * need roughly 0.5KB per GB. - */ -#ifdef CONFIG_X86_32 - start = 0x7000; -#else - start = 0x8000; + good_end = max_pfn_mapped << PAGE_SHIFT; #endif - base = memblock_find_in_range(start, max_pfn_mapped<<PAGE_SHIFT, - tables, PAGE_SIZE); + + base = memblock_find_in_range(start, good_end, tables, PAGE_SIZE); if (base == MEMBLOCK_ERROR) panic("Cannot find space for the kernel page tables"); @@ -279,25 +270,6 @@ unsigned long __init_refok init_memory_mapping(unsigned long start, load_cr3(swapper_pg_dir); #endif -#ifdef CONFIG_X86_64 - if (!after_bootmem && !start) { - pud_t *pud; - pmd_t *pmd; - - mmu_cr4_features = read_cr4(); - - /* - * _brk_end cannot change anymore, but it and _end may be - * located on different 2M pages. cleanup_highmap(), however, - * can only consider _end when it runs, so destroy any - * mappings beyond _brk_end here. - */ - pud = pud_offset(pgd_offset_k(_brk_end), _brk_end); - pmd = pmd_offset(pud, _brk_end - 1); - while (++pmd <= pmd_offset(pud, (unsigned long)_end - 1)) - pmd_clear(pmd); - } -#endif __flush_tlb_all(); if (!after_bootmem && e820_table_end > e820_table_start) diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 71a59296af80..194f2732ab77 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -333,12 +333,28 @@ static __ref void *alloc_low_page(unsigned long *phys) return adr; } +static __ref void *map_low_page(void *virt) +{ + void *adr; + unsigned long phys, left; + + if (after_bootmem) + return virt; + + phys = __pa(virt); + left = phys & (PAGE_SIZE - 1); + adr = early_memremap(phys & PAGE_MASK, PAGE_SIZE); + adr = (void *)(((unsigned long)adr) | left); + + return adr; +} + static __ref void unmap_low_page(void *adr) { if (after_bootmem) return; - early_iounmap(adr, PAGE_SIZE); + early_iounmap((void *)((unsigned long)adr & PAGE_MASK), PAGE_SIZE); } static unsigned long __meminit @@ -386,15 +402,6 @@ phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end, } static unsigned long __meminit -phys_pte_update(pmd_t *pmd, unsigned long address, unsigned long end, - pgprot_t prot) -{ - pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd); - - return phys_pte_init(pte, address, end, prot); -} - -static unsigned long __meminit phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, unsigned long page_size_mask, pgprot_t prot) { @@ -420,8 +427,10 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, if (pmd_val(*pmd)) { if (!pmd_large(*pmd)) { spin_lock(&init_mm.page_table_lock); - last_map_addr = phys_pte_update(pmd, address, + pte = map_low_page((pte_t *)pmd_page_vaddr(*pmd)); + last_map_addr = phys_pte_init(pte, address, end, prot); + unmap_low_page(pte); spin_unlock(&init_mm.page_table_lock); continue; } @@ -468,18 +477,6 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, } static unsigned long __meminit -phys_pmd_update(pud_t *pud, unsigned long address, unsigned long end, - unsigned long page_size_mask, pgprot_t prot) -{ - pmd_t *pmd = pmd_offset(pud, 0); - unsigned long last_map_addr; - - last_map_addr = phys_pmd_init(pmd, address, end, page_size_mask, prot); - __flush_tlb_all(); - return last_map_addr; -} - -static unsigned long __meminit phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end, unsigned long page_size_mask) { @@ -504,8 +501,11 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end, if (pud_val(*pud)) { if (!pud_large(*pud)) { - last_map_addr = phys_pmd_update(pud, addr, end, + pmd = map_low_page(pmd_offset(pud, 0)); + last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask, prot); + unmap_low_page(pmd); + __flush_tlb_all(); continue; } /* @@ -553,17 +553,6 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end, return last_map_addr; } -static unsigned long __meminit -phys_pud_update(pgd_t *pgd, unsigned long addr, unsigned long end, - unsigned long page_size_mask) -{ - pud_t *pud; - - pud = (pud_t *)pgd_page_vaddr(*pgd); - - return phys_pud_init(pud, addr, end, page_size_mask); -} - unsigned long __meminit kernel_physical_mapping_init(unsigned long start, unsigned long end, @@ -587,8 +576,10 @@ kernel_physical_mapping_init(unsigned long start, next = end; if (pgd_val(*pgd)) { - last_map_addr = phys_pud_update(pgd, __pa(start), + pud = map_low_page((pud_t *)pgd_page_vaddr(*pgd)); + last_map_addr = phys_pud_init(pud, __pa(start), __pa(end), page_size_mask); + unmap_low_page(pud); continue; } @@ -616,9 +607,63 @@ void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn, int acpi, int k8) { memblock_x86_register_active_regions(0, start_pfn, end_pfn); + init_memory_mapping_high(); } #endif +struct mapping_work_data { + unsigned long start; + unsigned long end; + unsigned long pfn_mapped; +}; + +static int __init_refok +mapping_work_fn(unsigned long start_pfn, unsigned long end_pfn, void *datax) +{ + struct mapping_work_data *data = datax; + unsigned long pfn_mapped; + unsigned long final_start, final_end; + + final_start = max_t(unsigned long, start_pfn<<PAGE_SHIFT, data->start); + final_end = min_t(unsigned long, end_pfn<<PAGE_SHIFT, data->end); + + if (final_end <= final_start) + return 0; + + pfn_mapped = init_memory_mapping(final_start, final_end); + + if (pfn_mapped > data->pfn_mapped) + data->pfn_mapped = pfn_mapped; + + return 0; +} + +static unsigned long __init_refok +init_memory_mapping_active_regions(unsigned long start, unsigned long end) +{ + struct mapping_work_data data; + + data.start = start; + data.end = end; + data.pfn_mapped = 0; + + work_with_active_regions(MAX_NUMNODES, mapping_work_fn, &data); + + return data.pfn_mapped; +} + +void __init_refok init_memory_mapping_high(void) +{ + if (max_pfn > max_low_pfn) { + max_pfn_mapped = init_memory_mapping_active_regions(1UL<<32, + max_pfn<<PAGE_SHIFT); + /* can we preserve max_low_pfn ? */ + max_low_pfn = max_pfn; + + memblock.current_limit = get_max_mapped(); + } +} + void __init paging_init(void) { unsigned long max_zone_pfns[MAX_NR_ZONES]; diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c index 3f9411ed3cdc..43ad3273561a 100644 --- a/arch/x86/mm/numa_64.c +++ b/arch/x86/mm/numa_64.c @@ -76,7 +76,7 @@ static int __init allocate_cachealigned_memnodemap(void) addr = 0x8000; nodemap_size = roundup(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES); - nodemap_addr = memblock_find_in_range(addr, max_pfn<<PAGE_SHIFT, + nodemap_addr = memblock_find_in_range(addr, get_max_mapped(), nodemap_size, L1_CACHE_BYTES); if (nodemap_addr == MEMBLOCK_ERROR) { printk(KERN_ERR @@ -566,11 +566,12 @@ static int __init numa_emulation(unsigned long start_pfn, * the e820 memory map. */ remove_all_active_ranges(); - for_each_node_mask(i, node_possible_map) { + for_each_node_mask(i, node_possible_map) memblock_x86_register_active_regions(i, nodes[i].start >> PAGE_SHIFT, nodes[i].end >> PAGE_SHIFT); + init_memory_mapping_high(); + for_each_node_mask(i, node_possible_map) setup_node_bootmem(i, nodes[i].start, nodes[i].end); - } setup_physnodes(addr, max_addr, acpi, amd); fake_physnodes(acpi, amd, num_nodes); numa_init_array(); @@ -626,6 +627,7 @@ void __init initmem_init(unsigned long start_pfn, unsigned long last_pfn, for (i = 0; i < nr_cpu_ids; i++) numa_set_node(i, 0); memblock_x86_register_active_regions(0, start_pfn, last_pfn); + init_memory_mapping_high(); setup_node_bootmem(0, start_pfn << PAGE_SHIFT, last_pfn << PAGE_SHIFT); } diff --git a/arch/x86/mm/srat_64.c b/arch/x86/mm/srat_64.c index 9a97261a2418..23498f8b09a2 100644 --- a/arch/x86/mm/srat_64.c +++ b/arch/x86/mm/srat_64.c @@ -444,6 +444,8 @@ int __init acpi_scan_nodes(unsigned long start, unsigned long end) return -1; } + init_memory_mapping_high(); + /* Account for nodes with cpus and no memory */ nodes_or(node_possible_map, nodes_parsed, cpu_nodes_parsed); diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 6acc724d5d8f..55272d7c3b0b 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -179,12 +179,8 @@ static void flush_tlb_others_ipi(const struct cpumask *cpumask, sender = this_cpu_read(tlb_vector_offset); f = &flush_state[sender]; - /* - * Could avoid this lock when - * num_online_cpus() <= NUM_INVALIDATE_TLB_VECTORS, but it is - * probably not worth checking this for a cache-hot lock. - */ - raw_spin_lock(&f->tlbstate_lock); + if (nr_cpu_ids > NUM_INVALIDATE_TLB_VECTORS) + raw_spin_lock(&f->tlbstate_lock); f->flush_mm = mm; f->flush_va = va; @@ -202,7 +198,8 @@ static void flush_tlb_others_ipi(const struct cpumask *cpumask, f->flush_mm = NULL; f->flush_va = 0; - raw_spin_unlock(&f->tlbstate_lock); + if (nr_cpu_ids > NUM_INVALIDATE_TLB_VECTORS) + raw_spin_unlock(&f->tlbstate_lock); } void native_flush_tlb_others(const struct cpumask *cpumask, |