diff options
Diffstat (limited to 'arch/riscv')
36 files changed, 468 insertions, 183 deletions
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 44377fd7860e..81b76d44725d 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -14,6 +14,8 @@ config RISCV def_bool y select ARCH_CLOCKSOURCE_INIT select ARCH_SUPPORTS_ATOMIC_RMW + select ARCH_SUPPORTS_DEBUG_PAGEALLOC if MMU + select ARCH_STACKWALK select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_VIRTUAL if MMU @@ -37,12 +39,12 @@ config RISCV select EDAC_SUPPORT select GENERIC_ARCH_TOPOLOGY if SMP select GENERIC_ATOMIC64 if !64BIT - select GENERIC_CLOCKEVENTS select GENERIC_EARLY_IOREMAP select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO select GENERIC_IOREMAP select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_SHOW + select GENERIC_LIB_DEVMEM_IS_ALLOWED select GENERIC_PCI_IOMAP select GENERIC_PTDUMP if MMU select GENERIC_SCHED_CLOCK @@ -68,6 +70,7 @@ config RISCV select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_GCC_PLUGINS select HAVE_GENERIC_VDSO if MMU && 64BIT + select HAVE_IRQ_TIME_ACCOUNTING select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS @@ -153,9 +156,6 @@ config ARCH_SELECT_MEMORY_MODEL config ARCH_WANT_GENERAL_HUGETLB def_bool y -config ARCH_SUPPORTS_DEBUG_PAGEALLOC - def_bool y - config SYS_SUPPORTS_HUGETLBFS depends on MMU def_bool y diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 0289a97325d1..8c29e553ef7f 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -96,5 +96,11 @@ $(BOOT_TARGETS): vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ @$(kecho) ' Kernel: $(boot)/$@ is ready' +Image.%: Image + $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ + zinstall install: $(Q)$(MAKE) $(build)=$(boot) $@ + +archclean: + $(Q)$(MAKE) $(clean)=$(boot) diff --git a/arch/riscv/boot/.gitignore b/arch/riscv/boot/.gitignore index 574c10f8ff68..90e66adb7de5 100644 --- a/arch/riscv/boot/.gitignore +++ b/arch/riscv/boot/.gitignore @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only Image -Image.gz +Image.* loader loader.lds +loader.bin diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile index c59fca695f9d..03404c84f971 100644 --- a/arch/riscv/boot/Makefile +++ b/arch/riscv/boot/Makefile @@ -18,7 +18,7 @@ KCOV_INSTRUMENT := n OBJCOPYFLAGS_Image :=-O binary -R .note -R .note.gnu.build-id -R .comment -S -targets := Image loader +targets := Image Image.* loader loader.o loader.lds loader.bin $(obj)/Image: vmlinux FORCE $(call if_changed,objcopy) diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h index 67c463812e2d..250defa06f3a 100644 --- a/arch/riscv/include/asm/mmu_context.h +++ b/arch/riscv/include/asm/mmu_context.h @@ -13,34 +13,16 @@ #include <linux/mm.h> #include <linux/sched.h> -static inline void enter_lazy_tlb(struct mm_struct *mm, - struct task_struct *task) -{ -} - -/* Initialize context-related info for a new mm_struct */ -static inline int init_new_context(struct task_struct *task, - struct mm_struct *mm) -{ - return 0; -} - -static inline void destroy_context(struct mm_struct *mm) -{ -} - void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *task); +#define activate_mm activate_mm static inline void activate_mm(struct mm_struct *prev, struct mm_struct *next) { switch_mm(prev, next, NULL); } -static inline void deactivate_mm(struct task_struct *task, - struct mm_struct *mm) -{ -} +#include <asm-generic/mmu_context.h> #endif /* _ASM_RISCV_MMU_CONTEXT_H */ diff --git a/arch/riscv/include/asm/pgtable-32.h b/arch/riscv/include/asm/pgtable-32.h index b0ab66e5fdb1..5b2e79e5bfa5 100644 --- a/arch/riscv/include/asm/pgtable-32.h +++ b/arch/riscv/include/asm/pgtable-32.h @@ -14,4 +14,6 @@ #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE - 1)) +#define MAX_POSSIBLE_PHYSMEM_BITS 34 + #endif /* _ASM_RISCV_PGTABLE_32_H */ diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 183f1f4b2ae6..41a72861987c 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -461,8 +461,6 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma, #define VMALLOC_START 0 #define VMALLOC_END TASK_SIZE -static inline void __kernel_map_pages(struct page *page, int numpages, int enable) {} - #endif /* !CONFIG_MMU */ #define kern_addr_valid(addr) (1) /* FIXME */ diff --git a/arch/riscv/include/asm/seccomp.h b/arch/riscv/include/asm/seccomp.h index bf7744ee3b3d..c7ee6a3507be 100644 --- a/arch/riscv/include/asm/seccomp.h +++ b/arch/riscv/include/asm/seccomp.h @@ -7,4 +7,14 @@ #include <asm-generic/seccomp.h> +#ifdef CONFIG_64BIT +# define SECCOMP_ARCH_NATIVE AUDIT_ARCH_RISCV64 +# define SECCOMP_ARCH_NATIVE_NR NR_syscalls +# define SECCOMP_ARCH_NATIVE_NAME "riscv64" +#else /* !CONFIG_64BIT */ +# define SECCOMP_ARCH_NATIVE AUDIT_ARCH_RISCV32 +# define SECCOMP_ARCH_NATIVE_NR NR_syscalls +# define SECCOMP_ARCH_NATIVE_NAME "riscv32" +#endif + #endif /* _ASM_SECCOMP_H */ diff --git a/arch/riscv/include/asm/sections.h b/arch/riscv/include/asm/sections.h index 3a9971b1210f..1595c5b60cfd 100644 --- a/arch/riscv/include/asm/sections.h +++ b/arch/riscv/include/asm/sections.h @@ -9,5 +9,7 @@ extern char _start[]; extern char _start_kernel[]; +extern char __init_data_begin[], __init_data_end[]; +extern char __init_text_begin[], __init_text_end[]; #endif /* __ASM_SECTIONS_H */ diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index 4c5bae7ca01c..211eb8244a45 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -15,15 +15,20 @@ int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); int set_memory_x(unsigned long addr, int numpages); int set_memory_nx(unsigned long addr, int numpages); +int set_memory_rw_nx(unsigned long addr, int numpages); +void protect_kernel_text_data(void); #else static inline int set_memory_ro(unsigned long addr, int numpages) { return 0; } static inline int set_memory_rw(unsigned long addr, int numpages) { return 0; } static inline int set_memory_x(unsigned long addr, int numpages) { return 0; } static inline int set_memory_nx(unsigned long addr, int numpages) { return 0; } +static inline void protect_kernel_text_data(void) {}; +static inline int set_memory_rw_nx(unsigned long addr, int numpages) { return 0; } #endif int set_direct_map_invalid_noflush(struct page *page); int set_direct_map_default_noflush(struct page *page); +bool kernel_page_present(struct page *page); #endif /* __ASSEMBLY__ */ diff --git a/arch/riscv/include/asm/stacktrace.h b/arch/riscv/include/asm/stacktrace.h new file mode 100644 index 000000000000..470a65c4ccdc --- /dev/null +++ b/arch/riscv/include/asm/stacktrace.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_STACKTRACE_H +#define _ASM_RISCV_STACKTRACE_H + +#include <linux/sched.h> +#include <asm/ptrace.h> + +struct stackframe { + unsigned long fp; + unsigned long ra; +}; + +extern void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, + bool (*fn)(void *, unsigned long), void *arg); + +#endif /* _ASM_RISCV_STACKTRACE_H */ diff --git a/arch/riscv/include/asm/string.h b/arch/riscv/include/asm/string.h index 924af13f8555..5477e7ecb6e1 100644 --- a/arch/riscv/include/asm/string.h +++ b/arch/riscv/include/asm/string.h @@ -12,16 +12,16 @@ #define __HAVE_ARCH_MEMSET extern asmlinkage void *memset(void *, int, size_t); extern asmlinkage void *__memset(void *, int, size_t); - #define __HAVE_ARCH_MEMCPY extern asmlinkage void *memcpy(void *, const void *, size_t); extern asmlinkage void *__memcpy(void *, const void *, size_t); - +#define __HAVE_ARCH_MEMMOVE +extern asmlinkage void *memmove(void *, const void *, size_t); +extern asmlinkage void *__memmove(void *, const void *, size_t); /* For those files which don't want to check by kasan. */ #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) - #define memcpy(dst, src, len) __memcpy(dst, src, len) #define memset(s, c, n) __memset(s, c, n) - +#define memmove(dst, src, len) __memmove(dst, src, len) #endif #endif /* _ASM_RISCV_STRING_H */ diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index a390711129de..97bf5a1575d2 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -74,6 +74,7 @@ struct thread_info { #define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */ #define TIF_SYSCALL_AUDIT 7 /* syscall auditing */ #define TIF_SECCOMP 8 /* syscall secure computing */ +#define TIF_NOTIFY_SIGNAL 9 /* signal notifications exist */ #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) @@ -82,9 +83,11 @@ struct thread_info { #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_SECCOMP (1 << TIF_SECCOMP) +#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) #define _TIF_WORK_MASK \ - (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED) + (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \ + _TIF_NOTIFY_SIGNAL) #define _TIF_SYSCALL_WORK \ (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \ diff --git a/arch/riscv/include/asm/timex.h b/arch/riscv/include/asm/timex.h index ab104905d4db..81de51e6aa32 100644 --- a/arch/riscv/include/asm/timex.h +++ b/arch/riscv/include/asm/timex.h @@ -60,6 +60,8 @@ static inline u32 get_cycles_hi(void) } #define get_cycles_hi get_cycles_hi +#endif /* !CONFIG_RISCV_M_MODE */ + #ifdef CONFIG_64BIT static inline u64 get_cycles64(void) { @@ -79,8 +81,6 @@ static inline u64 get_cycles64(void) } #endif /* CONFIG_64BIT */ -#endif /* !CONFIG_RISCV_M_MODE */ - #define ARCH_HAS_READ_CURRENT_TIMER static inline int read_current_timer(unsigned long *timer_val) { diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index c47e6b35c551..824b2c9da75b 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -476,7 +476,7 @@ do { \ do { \ long __kr_err; \ \ - __put_user_nocheck(*((type *)(dst)), (type *)(src), __kr_err); \ + __put_user_nocheck(*((type *)(src)), (type *)(dst), __kr_err); \ if (unlikely(__kr_err)) \ goto err_label; \ } while (0) diff --git a/arch/riscv/include/asm/vdso/processor.h b/arch/riscv/include/asm/vdso/processor.h index 82a5693b1861..134388cbaaa1 100644 --- a/arch/riscv/include/asm/vdso/processor.h +++ b/arch/riscv/include/asm/vdso/processor.h @@ -4,6 +4,8 @@ #ifndef __ASSEMBLY__ +#include <asm/barrier.h> + static inline void cpu_relax(void) { #ifdef __riscv_muldiv diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index fa896c5f7ccb..f6caf4d9ca15 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -56,5 +56,3 @@ obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-$(CONFIG_EFI) += efi.o - -clean: diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index db203442c08f..b79ffa3561fd 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -11,6 +11,8 @@ #include <asm/thread_info.h> #include <asm/ptrace.h> +void asm_offsets(void); + void asm_offsets(void) { OFFSET(TASK_THREAD_RA, task_struct, thread.ra); diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 99e12faa5498..765b62434f30 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2013 Linaro Limited * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 11e2a4fe66e0..16e9941900c4 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -35,6 +35,10 @@ ENTRY(_start) .word 0 #endif .balign 8 +#ifdef CONFIG_RISCV_M_MODE + /* Image load offset (0MB) from start of RAM for M-mode */ + .dword 0 +#else #if __riscv_xlen == 64 /* Image load offset(2MB) from start of RAM */ .dword 0x200000 @@ -42,6 +46,7 @@ ENTRY(_start) /* Image load offset(4MB) from start of RAM */ .dword 0x400000 #endif +#endif /* Effective size of kernel image */ .dword _end - _start .dword __HEAD_FLAGS @@ -177,7 +182,6 @@ setup_trap_vector: END(_start) - __INIT ENTRY(_start_kernel) /* Mask all interrupts */ csrw CSR_IE, zero diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c index cf190197a22f..0bb1854dce83 100644 --- a/arch/riscv/kernel/perf_callchain.c +++ b/arch/riscv/kernel/perf_callchain.c @@ -4,11 +4,7 @@ #include <linux/perf_event.h> #include <linux/uaccess.h> -/* Kernel callchain */ -struct stackframe { - unsigned long fp; - unsigned long ra; -}; +#include <asm/stacktrace.h> /* * Get the return address for a single stackframe and return a pointer to the @@ -74,13 +70,11 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, fp = user_backtrace(entry, fp, 0); } -bool fill_callchain(unsigned long pc, void *entry) +static bool fill_callchain(void *entry, unsigned long pc) { return perf_callchain_store(entry, pc); } -void notrace walk_stackframe(struct task_struct *task, - struct pt_regs *regs, bool (*fn)(unsigned long, void *), void *arg); void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { diff --git a/arch/riscv/kernel/perf_regs.c b/arch/riscv/kernel/perf_regs.c index 04a38fbeb9c7..fd304a248de6 100644 --- a/arch/riscv/kernel/perf_regs.c +++ b/arch/riscv/kernel/perf_regs.c @@ -36,8 +36,7 @@ u64 perf_reg_abi(struct task_struct *task) } void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs, - struct pt_regs *regs_user_copy) + struct pt_regs *regs) { regs_user->regs = task_pt_regs(current); regs_user->abi = perf_reg_abi(current); diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 19225ec65db6..dd5f985b1f40 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -36,7 +36,7 @@ extern asmlinkage void ret_from_kernel_thread(void); void arch_cpu_idle(void) { wait_for_interrupt(); - local_irq_enable(); + raw_local_irq_enable(); } void show_regs(struct pt_regs *regs) diff --git a/arch/riscv/kernel/riscv_ksyms.c b/arch/riscv/kernel/riscv_ksyms.c index 450492e1cb4e..5ab1c7e1a6ed 100644 --- a/arch/riscv/kernel/riscv_ksyms.c +++ b/arch/riscv/kernel/riscv_ksyms.c @@ -11,5 +11,7 @@ */ EXPORT_SYMBOL(memset); EXPORT_SYMBOL(memcpy); +EXPORT_SYMBOL(memmove); EXPORT_SYMBOL(__memset); EXPORT_SYMBOL(__memcpy); +EXPORT_SYMBOL(__memmove); diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index c424cc6dd833..1d85e9bf783c 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -4,6 +4,8 @@ * Chen Liqin <liqin.chen@sunplusct.com> * Lennox Wu <lennox.wu@sunplusct.com> * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2020 FORTH-ICS/CARV + * Nick Kossifidis <mick@ics.forth.gr> */ #include <linux/init.h> @@ -22,6 +24,7 @@ #include <asm/cpu_ops.h> #include <asm/early_ioremap.h> #include <asm/setup.h> +#include <asm/set_memory.h> #include <asm/sections.h> #include <asm/sbi.h> #include <asm/tlbflush.h> @@ -51,6 +54,163 @@ atomic_t hart_lottery __section(".sdata"); unsigned long boot_cpu_hartid; static DEFINE_PER_CPU(struct cpu, cpu_devices); +/* + * Place kernel memory regions on the resource tree so that + * kexec-tools can retrieve them from /proc/iomem. While there + * also add "System RAM" regions for compatibility with other + * archs, and the rest of the known regions for completeness. + */ +static struct resource code_res = { .name = "Kernel code", }; +static struct resource data_res = { .name = "Kernel data", }; +static struct resource rodata_res = { .name = "Kernel rodata", }; +static struct resource bss_res = { .name = "Kernel bss", }; + +static int __init add_resource(struct resource *parent, + struct resource *res) +{ + int ret = 0; + + ret = insert_resource(parent, res); + if (ret < 0) { + pr_err("Failed to add a %s resource at %llx\n", + res->name, (unsigned long long) res->start); + return ret; + } + + return 1; +} + +static int __init add_kernel_resources(struct resource *res) +{ + int ret = 0; + + /* + * The memory region of the kernel image is continuous and + * was reserved on setup_bootmem, find it here and register + * it as a resource, then register the various segments of + * the image as child nodes + */ + if (!(res->start <= code_res.start && res->end >= data_res.end)) + return 0; + + res->name = "Kernel image"; + res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + + /* + * We removed a part of this region on setup_bootmem so + * we need to expand the resource for the bss to fit in. + */ + res->end = bss_res.end; + + ret = add_resource(&iomem_resource, res); + if (ret < 0) + return ret; + + ret = add_resource(res, &code_res); + if (ret < 0) + return ret; + + ret = add_resource(res, &rodata_res); + if (ret < 0) + return ret; + + ret = add_resource(res, &data_res); + if (ret < 0) + return ret; + + ret = add_resource(res, &bss_res); + + return ret; +} + +static void __init init_resources(void) +{ + struct memblock_region *region = NULL; + struct resource *res = NULL; + int ret = 0; + + code_res.start = __pa_symbol(_text); + code_res.end = __pa_symbol(_etext) - 1; + code_res.flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + + rodata_res.start = __pa_symbol(__start_rodata); + rodata_res.end = __pa_symbol(__end_rodata) - 1; + rodata_res.flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + + data_res.start = __pa_symbol(_data); + data_res.end = __pa_symbol(_edata) - 1; + data_res.flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + + bss_res.start = __pa_symbol(__bss_start); + bss_res.end = __pa_symbol(__bss_stop) - 1; + bss_res.flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + + /* + * Start by adding the reserved regions, if they overlap + * with /memory regions, insert_resource later on will take + * care of it. + */ + for_each_reserved_mem_region(region) { + res = memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES); + if (!res) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(struct resource)); + + res->name = "Reserved"; + res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; + res->start = __pfn_to_phys(memblock_region_reserved_base_pfn(region)); + res->end = __pfn_to_phys(memblock_region_reserved_end_pfn(region)) - 1; + + ret = add_kernel_resources(res); + if (ret < 0) + goto error; + else if (ret) + continue; + + /* + * Ignore any other reserved regions within + * system memory. + */ + if (memblock_is_memory(res->start)) + continue; + + ret = add_resource(&iomem_resource, res); + if (ret < 0) + goto error; + } + + /* Add /memory regions to the resource tree */ + for_each_mem_region(region) { + res = memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES); + if (!res) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(struct resource)); + + if (unlikely(memblock_is_nomap(region))) { + res->name = "Reserved"; + res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; + } else { + res->name = "System RAM"; + res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + } + + res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region)); + res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1; + + ret = add_resource(&iomem_resource, res); + if (ret < 0) + goto error; + } + + return; + + error: + memblock_free((phys_addr_t) res, sizeof(struct resource)); + /* Better an empty resource tree than an inconsistent one */ + release_child_resources(&iomem_resource); +} + + static void __init parse_dtb(void) { /* Early scan of device tree from init memory */ @@ -75,11 +235,13 @@ void __init setup_arch(char **cmdline_p) *cmdline_p = boot_command_line; early_ioremap_setup(); + jump_label_init(); parse_early_param(); efi_init(); setup_bootmem(); paging_init(); + init_resources(); #if IS_ENABLED(CONFIG_BUILTIN_DTB) unflatten_and_copy_device_tree(); #else @@ -89,6 +251,11 @@ void __init setup_arch(char **cmdline_p) pr_err("No DTB found in kernel mappings\n"); #endif + if (IS_ENABLED(CONFIG_RISCV_SBI)) + sbi_init(); + + if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) + protect_kernel_text_data(); #ifdef CONFIG_SWIOTLB swiotlb_init(1); #endif @@ -97,10 +264,6 @@ void __init setup_arch(char **cmdline_p) kasan_init(); #endif -#if IS_ENABLED(CONFIG_RISCV_SBI) - sbi_init(); -#endif - #ifdef CONFIG_SMP setup_smp(); #endif @@ -122,3 +285,12 @@ static int __init topology_init(void) return 0; } subsys_initcall(topology_init); + +void free_initmem(void) +{ + unsigned long init_begin = (unsigned long)__init_begin; + unsigned long init_end = (unsigned long)__init_end; + + set_memory_rw_nx(init_begin, (init_end - init_begin) >> PAGE_SHIFT); + free_initmem_default(POISON_FREE_INITMEM); +} diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index bc6841867b51..469aef8ed922 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -310,7 +310,7 @@ asmlinkage __visible void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags) { /* Handle pending signal delivery */ - if (thread_info_flags & _TIF_SIGPENDING) + if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) do_signal(regs); if (thread_info_flags & _TIF_NOTIFY_RESUME) diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index 595342910c3f..48b870a685b3 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -12,17 +12,14 @@ #include <linux/stacktrace.h> #include <linux/ftrace.h> +#include <asm/stacktrace.h> + register unsigned long sp_in_global __asm__("sp"); #ifdef CONFIG_FRAME_POINTER -struct stackframe { - unsigned long fp; - unsigned long ra; -}; - void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, - bool (*fn)(unsigned long, void *), void *arg) + bool (*fn)(void *, unsigned long), void *arg) { unsigned long fp, sp, pc; @@ -46,7 +43,7 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, unsigned long low, high; struct stackframe *frame; - if (unlikely(!__kernel_text_address(pc) || fn(pc, arg))) + if (unlikely(!__kernel_text_address(pc) || !fn(arg, pc))) break; /* Validate frame pointer */ @@ -66,7 +63,7 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, #else /* !CONFIG_FRAME_POINTER */ void notrace walk_stackframe(struct task_struct *task, - struct pt_regs *regs, bool (*fn)(unsigned long, void *), void *arg) + struct pt_regs *regs, bool (*fn)(void *, unsigned long), void *arg) { unsigned long sp, pc; unsigned long *ksp; @@ -88,7 +85,7 @@ void notrace walk_stackframe(struct task_struct *task, ksp = (unsigned long *)sp; while (!kstack_end(ksp)) { - if (__kernel_text_address(pc) && unlikely(fn(pc, arg))) + if (__kernel_text_address(pc) && unlikely(!fn(arg, pc))) break; pc = (*ksp++) - 0x4; } @@ -96,13 +93,12 @@ void notrace walk_stackframe(struct task_struct *task, #endif /* CONFIG_FRAME_POINTER */ - -static bool print_trace_address(unsigned long pc, void *arg) +static bool print_trace_address(void *arg, unsigned long pc) { const char *loglvl = arg; print_ip_sym(loglvl, pc); - return false; + return true; } void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) @@ -111,14 +107,14 @@ void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) walk_stackframe(task, NULL, print_trace_address, (void *)loglvl); } -static bool save_wchan(unsigned long pc, void *arg) +static bool save_wchan(void *arg, unsigned long pc) { if (!in_sched_functions(pc)) { unsigned long *p = arg; *p = pc; - return true; + return false; } - return false; + return true; } unsigned long get_wchan(struct task_struct *task) @@ -130,42 +126,12 @@ unsigned long get_wchan(struct task_struct *task) return pc; } - #ifdef CONFIG_STACKTRACE -static bool __save_trace(unsigned long pc, void *arg, bool nosched) -{ - struct stack_trace *trace = arg; - - if (unlikely(nosched && in_sched_functions(pc))) - return false; - if (unlikely(trace->skip > 0)) { - trace->skip--; - return false; - } - - trace->entries[trace->nr_entries++] = pc; - return (trace->nr_entries >= trace->max_entries); -} - -static bool save_trace(unsigned long pc, void *arg) -{ - return __save_trace(pc, arg, false); -} - -/* - * Save stack-backtrace addresses into a stack_trace buffer. - */ -void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) -{ - walk_stackframe(tsk, NULL, save_trace, trace); -} -EXPORT_SYMBOL_GPL(save_stack_trace_tsk); - -void save_stack_trace(struct stack_trace *trace) +void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, + struct task_struct *task, struct pt_regs *regs) { - save_stack_trace_tsk(NULL, trace); + walk_stackframe(task, regs, consume_entry, cookie); } -EXPORT_SYMBOL_GPL(save_stack_trace); #endif /* CONFIG_STACKTRACE */ diff --git a/arch/riscv/kernel/vdso/.gitignore b/arch/riscv/kernel/vdso/.gitignore index 11ebee9e4c1d..3a19def868ec 100644 --- a/arch/riscv/kernel/vdso/.gitignore +++ b/arch/riscv/kernel/vdso/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only vdso.lds *.tmp +vdso-syms.S diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 7d6a94d45ec9..0cfd6da784f8 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -43,19 +43,14 @@ $(obj)/vdso.o: $(obj)/vdso.so SYSCFLAGS_vdso.so.dbg = $(c_flags) $(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) FORCE $(call if_changed,vdsold) +SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ + -Wl,--build-id=sha1 -Wl,--hash-style=both # We also create a special relocatable object that should mirror the symbol # table and layout of the linked DSO. With ld --just-symbols we can then # refer to these symbols in the kernel code rather than hand-coded addresses. - -SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ - -Wl,--build-id=sha1 -Wl,--hash-style=both -$(obj)/vdso-dummy.o: $(src)/vdso.lds $(obj)/rt_sigreturn.o FORCE - $(call if_changed,vdsold) - -LDFLAGS_vdso-syms.o := -r --just-symbols -$(obj)/vdso-syms.o: $(obj)/vdso-dummy.o FORCE - $(call if_changed,ld) +$(obj)/vdso-syms.S: $(obj)/vdso.so FORCE + $(call if_changed,so2s) # strip rule for the .so file $(obj)/%.so: OBJCOPYFLAGS := -S @@ -73,6 +68,11 @@ quiet_cmd_vdsold = VDSOLD $@ $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ rm $@.tmp +# Extracts symbol offsets from the VDSO, converting them into an assembly file +# that contains the same symbols at the same offsets. +quiet_cmd_so2s = SO2S $@ + cmd_so2s = $(NM) -D $< | $(srctree)/$(src)/so2s.sh > $@ + # install commands for the unstripped file quiet_cmd_vdso_install = INSTALL $@ cmd_vdso_install = cp $(obj)/$@.dbg $(MODLIB)/vdso/$@ diff --git a/arch/riscv/kernel/vdso/so2s.sh b/arch/riscv/kernel/vdso/so2s.sh new file mode 100755 index 000000000000..e64cb6d9440e --- /dev/null +++ b/arch/riscv/kernel/vdso/so2s.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2020 Palmer Dabbelt <palmerdabbelt@google.com> + +sed 's!\([0-9a-f]*\) T \([a-z0-9_]*\)\(@@LINUX_4.15\)*!.global \2\n.set \2,0x\1!' \ +| grep '^\.' diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 3ffbd6cbdb86..de03cb22d0e9 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -29,8 +29,30 @@ SECTIONS HEAD_TEXT_SECTION . = ALIGN(PAGE_SIZE); + .text : { + _text = .; + _stext = .; + TEXT_TEXT + SCHED_TEXT + CPUIDLE_TEXT + LOCK_TEXT + KPROBES_TEXT + ENTRY_TEXT + IRQENTRY_TEXT + SOFTIRQENTRY_TEXT + *(.fixup) + _etext = .; + } + + . = ALIGN(SECTION_ALIGN); __init_begin = .; - INIT_TEXT_SECTION(PAGE_SIZE) + __init_text_begin = .; + .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) ALIGN(SECTION_ALIGN) { \ + _sinittext = .; \ + INIT_TEXT \ + _einittext = .; \ + } + . = ALIGN(8); __soc_early_init_table : { __soc_early_init_table_start = .; @@ -47,35 +69,28 @@ SECTIONS { EXIT_TEXT } + + __init_text_end = .; + . = ALIGN(SECTION_ALIGN); +#ifdef CONFIG_EFI + . = ALIGN(PECOFF_SECTION_ALIGNMENT); + __pecoff_text_end = .; +#endif + /* Start of init data section */ + __init_data_begin = .; + INIT_DATA_SECTION(16) .exit.data : { EXIT_DATA } PERCPU_SECTION(L1_CACHE_BYTES) - __init_end = .; - . = ALIGN(SECTION_ALIGN); - .text : { - _text = .; - _stext = .; - TEXT_TEXT - SCHED_TEXT - CPUIDLE_TEXT - LOCK_TEXT - KPROBES_TEXT - ENTRY_TEXT - IRQENTRY_TEXT - SOFTIRQENTRY_TEXT - *(.fixup) - _etext = .; + .rel.dyn : { + *(.rel.dyn*) } -#ifdef CONFIG_EFI - . = ALIGN(PECOFF_SECTION_ALIGNMENT); - __pecoff_text_end = .; -#endif - - INIT_DATA_SECTION(16) + __init_data_end = .; + __init_end = .; /* Start of data section */ _sdata = .; @@ -105,10 +120,6 @@ SECTIONS BSS_SECTION(PAGE_SIZE, PAGE_SIZE, 0) - .rel.dyn : { - *(.rel.dyn*) - } - #ifdef CONFIG_EFI . = ALIGN(PECOFF_SECTION_ALIGNMENT); __pecoff_data_virt_size = ABSOLUTE(. - __pecoff_text_end); diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 47e7a8204460..ac6171e9c19e 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -2,5 +2,6 @@ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o +lib-y += memmove.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o diff --git a/arch/riscv/lib/memmove.S b/arch/riscv/lib/memmove.S new file mode 100644 index 000000000000..07d1d2152ba5 --- /dev/null +++ b/arch/riscv/lib/memmove.S @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +ENTRY(__memmove) +WEAK(memmove) + move t0, a0 + move t1, a1 + + beq a0, a1, exit_memcpy + beqz a2, exit_memcpy + srli t2, a2, 0x2 + + slt t3, a0, a1 + beqz t3, do_reverse + + andi a2, a2, 0x3 + li t4, 1 + beqz t2, byte_copy + +word_copy: + lw t3, 0(a1) + addi t2, t2, -1 + addi a1, a1, 4 + sw t3, 0(a0) + addi a0, a0, 4 + bnez t2, word_copy + beqz a2, exit_memcpy + j byte_copy + +do_reverse: + add a0, a0, a2 + add a1, a1, a2 + andi a2, a2, 0x3 + li t4, -1 + beqz t2, reverse_byte_copy + +reverse_word_copy: + addi a1, a1, -4 + addi t2, t2, -1 + lw t3, 0(a1) + addi a0, a0, -4 + sw t3, 0(a0) + bnez t2, reverse_word_copy + beqz a2, exit_memcpy + +reverse_byte_copy: + addi a0, a0, -1 + addi a1, a1, -1 + +byte_copy: + lb t3, 0(a1) + addi a2, a2, -1 + sb t3, 0(a0) + add a1, a1, t4 + add a0, a0, t4 + bnez a2, byte_copy + +exit_memcpy: + move a0, t0 + move a1, t1 + ret +END(__memmove) diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 1359e21c0c62..3c8b9e433c67 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -86,6 +86,7 @@ static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long a pmd_t *pmd, *pmd_k; pte_t *pte_k; int index; + unsigned long pfn; /* User mode accesses just cause a SIGSEGV */ if (user_mode(regs)) @@ -100,7 +101,8 @@ static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long a * of a task switch. */ index = pgd_index(addr); - pgd = (pgd_t *)pfn_to_virt(csr_read(CSR_SATP)) + index; + pfn = csr_read(CSR_SATP) & SATP_PPN; + pgd = (pgd_t *)pfn_to_virt(pfn) + index; pgd_k = init_mm.pgd + index; if (!pgd_present(*pgd_k)) { diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index ea933b789a88..13ba533f462b 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -13,6 +13,7 @@ #include <linux/of_fdt.h> #include <linux/libfdt.h> #include <linux/set_memory.h> +#include <linux/dma-map-ops.h> #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -41,13 +42,14 @@ struct pt_alloc_ops { #endif }; +static phys_addr_t dma32_phys_limit __ro_after_init; + static void __init zone_sizes_init(void) { unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; #ifdef CONFIG_ZONE_DMA32 - max_zone_pfns[ZONE_DMA32] = PFN_DOWN(min(4UL * SZ_1G, - (unsigned long) PFN_PHYS(max_low_pfn))); + max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); #endif max_zone_pfns[ZONE_NORMAL] = max_low_pfn; @@ -154,9 +156,8 @@ disable: void __init setup_bootmem(void) { - phys_addr_t mem_size = 0; - phys_addr_t total_mem = 0; - phys_addr_t mem_start, start, end = 0; + phys_addr_t mem_start = 0; + phys_addr_t start, end = 0; phys_addr_t vmlinux_end = __pa_symbol(&_end); phys_addr_t vmlinux_start = __pa_symbol(&_start); u64 i; @@ -164,27 +165,25 @@ void __init setup_bootmem(void) /* Find the memory region containing the kernel */ for_each_mem_range(i, &start, &end) { phys_addr_t size = end - start; - if (!total_mem) + if (!mem_start) mem_start = start; if (start <= vmlinux_start && vmlinux_end <= end) BUG_ON(size == 0); - total_mem = total_mem + size; } /* - * Remove memblock from the end of usable area to the - * end of region + * The maximal physical memory size is -PAGE_OFFSET. + * Make sure that any memory beyond mem_start + (-PAGE_OFFSET) is removed + * as it is unusable by kernel. */ - mem_size = min(total_mem, (phys_addr_t)-PAGE_OFFSET); - if (mem_start + mem_size < end) - memblock_remove(mem_start + mem_size, - end - mem_start - mem_size); + memblock_enforce_memory_limit(mem_start - PAGE_OFFSET); /* Reserve from the start of the kernel to the end of the kernel */ memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); max_pfn = PFN_DOWN(memblock_end_of_DRAM()); max_low_pfn = max_pfn; + dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); set_max_mapnr(max_low_pfn); #ifdef CONFIG_BLK_DEV_INITRD @@ -198,6 +197,7 @@ void __init setup_bootmem(void) memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); early_init_fdt_scan_reserved_mem(); + dma_contiguous_reserve(dma32_phys_limit); memblock_allow_resize(); memblock_dump_all(); } @@ -297,6 +297,7 @@ pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; #define NUM_EARLY_PMDS (1UL + MAX_EARLY_MAPPING_SIZE / PGDIR_SIZE) #endif pmd_t early_pmd[PTRS_PER_PMD * NUM_EARLY_PMDS] __initdata __aligned(PAGE_SIZE); +pmd_t early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) { @@ -494,6 +495,18 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) load_pa + (va - PAGE_OFFSET), map_size, PAGE_KERNEL_EXEC); +#ifndef __PAGETABLE_PMD_FOLDED + /* Setup early PMD for DTB */ + create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, + (uintptr_t)early_dtb_pmd, PGDIR_SIZE, PAGE_TABLE); + /* Create two consecutive PMD mappings for FDT early scan */ + pa = dtb_pa & ~(PMD_SIZE - 1); + create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA, + pa, PMD_SIZE, PAGE_KERNEL); + create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA + PMD_SIZE, + pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL); + dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PMD_SIZE - 1)); +#else /* Create two consecutive PGD mappings for FDT early scan */ pa = dtb_pa & ~(PGDIR_SIZE - 1); create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, @@ -501,6 +514,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA + PGDIR_SIZE, pa + PGDIR_SIZE, PGDIR_SIZE, PAGE_KERNEL); dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PGDIR_SIZE - 1)); +#endif dtb_early_pa = dtb_pa; /* @@ -608,48 +622,33 @@ static inline void setup_vm_final(void) #endif /* CONFIG_MMU */ #ifdef CONFIG_STRICT_KERNEL_RWX -void mark_rodata_ro(void) +void protect_kernel_text_data(void) { - unsigned long text_start = (unsigned long)_text; - unsigned long text_end = (unsigned long)_etext; + unsigned long text_start = (unsigned long)_start; + unsigned long init_text_start = (unsigned long)__init_text_begin; + unsigned long init_data_start = (unsigned long)__init_data_begin; unsigned long rodata_start = (unsigned long)__start_rodata; unsigned long data_start = (unsigned long)_data; unsigned long max_low = (unsigned long)(__va(PFN_PHYS(max_low_pfn))); - set_memory_ro(text_start, (text_end - text_start) >> PAGE_SHIFT); - set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); + set_memory_ro(text_start, (init_text_start - text_start) >> PAGE_SHIFT); + set_memory_ro(init_text_start, (init_data_start - init_text_start) >> PAGE_SHIFT); + set_memory_nx(init_data_start, (rodata_start - init_data_start) >> PAGE_SHIFT); + /* rodata section is marked readonly in mark_rodata_ro */ set_memory_nx(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); set_memory_nx(data_start, (max_low - data_start) >> PAGE_SHIFT); - - debug_checkwx(); } -#endif -static void __init resource_init(void) +void mark_rodata_ro(void) { - struct memblock_region *region; - - for_each_mem_region(region) { - struct resource *res; - - res = memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES); - if (!res) - panic("%s: Failed to allocate %zu bytes\n", __func__, - sizeof(struct resource)); + unsigned long rodata_start = (unsigned long)__start_rodata; + unsigned long data_start = (unsigned long)_data; - if (memblock_is_nomap(region)) { - res->name = "reserved"; - res->flags = IORESOURCE_MEM; - } else { - res->name = "System RAM"; - res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; - } - res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region)); - res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1; + set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); - request_resource(&iomem_resource, res); - } + debug_checkwx(); } +#endif void __init paging_init(void) { @@ -657,7 +656,6 @@ void __init paging_init(void) sparse_init(); setup_zero_page(); zone_sizes_init(); - resource_init(); } #ifdef CONFIG_SPARSEMEM_VMEMMAP diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index 19fecb362d81..5e49e4b4a4cc 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -128,6 +128,12 @@ static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, return ret; } +int set_memory_rw_nx(unsigned long addr, int numpages) +{ + return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE), + __pgprot(_PAGE_EXEC)); +} + int set_memory_ro(unsigned long addr, int numpages) { return __set_memory(addr, numpages, __pgprot(_PAGE_READ), @@ -184,6 +190,7 @@ int set_direct_map_default_noflush(struct page *page) return ret; } +#ifdef CONFIG_DEBUG_PAGEALLOC void __kernel_map_pages(struct page *page, int numpages, int enable) { if (!debug_pagealloc_enabled()) @@ -196,3 +203,33 @@ void __kernel_map_pages(struct page *page, int numpages, int enable) __set_memory((unsigned long)page_address(page), numpages, __pgprot(0), __pgprot(_PAGE_PRESENT)); } +#endif + +bool kernel_page_present(struct page *page) +{ + unsigned long addr = (unsigned long)page_address(page); + pgd_t *pgd; + pud_t *pud; + p4d_t *p4d; + pmd_t *pmd; + pte_t *pte; + + pgd = pgd_offset_k(addr); + if (!pgd_present(*pgd)) + return false; + + p4d = p4d_offset(pgd, addr); + if (!p4d_present(*p4d)) + return false; + + pud = pud_offset(p4d, addr); + if (!pud_present(*pud)) + return false; + + pmd = pmd_offset(pud, addr); + if (!pmd_present(*pmd)) + return false; + + pte = pte_offset_kernel(pmd, addr); + return pte_present(*pte); +} |