From fd200632d09dcea9fd67714241155b7635505cd6 Mon Sep 17 00:00:00 2001 From: Jinyang He Date: Sat, 25 Feb 2023 15:52:56 +0800 Subject: LoongArch: Fix Chinese comma in cpu.h Fix Chinese comma introduced by accident in cpu.h. Signed-off-by: Jinyang He Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/cpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/loongarch') diff --git a/arch/loongarch/include/asm/cpu.h b/arch/loongarch/include/asm/cpu.h index 754f28506791..c3da91759472 100644 --- a/arch/loongarch/include/asm/cpu.h +++ b/arch/loongarch/include/asm/cpu.h @@ -36,7 +36,7 @@ #define PRID_SERIES_LA132 0x8000 /* Loongson 32bit */ #define PRID_SERIES_LA264 0xa000 /* Loongson 64bit, 2-issue */ -#define PRID_SERIES_LA364 0xb000 /* Loongson 64bit，3-issue */ +#define PRID_SERIES_LA364 0xb000 /* Loongson 64bit, 3-issue */ #define PRID_SERIES_LA464 0xc000 /* Loongson 64bit, 4-issue */ #define PRID_SERIES_LA664 0xd000 /* Loongson 64bit, 6-issue */ -- cgit v1.2.3 From bb7a78e343468873bf00b2b181fcfd3c02d8cb56 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Sat, 25 Feb 2023 15:52:56 +0800 Subject: LoongArch: Only call get_timer_irq() once in constant_clockevent_init() Under CONFIG_DEBUG_ATOMIC_SLEEP=y and CONFIG_DEBUG_PREEMPT=y, we can see the following messages on LoongArch, this is because using might_sleep() in preemption disable context. [ 0.001127] smp: Bringing up secondary CPUs ... [ 0.001222] Booting CPU#1... [ 0.001244] 64-bit Loongson Processor probed (LA464 Core) [ 0.001247] CPU1 revision is: 0014c012 (Loongson-64bit) [ 0.001250] FPU1 revision is: 00000000 [ 0.001252] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:283 [ 0.001255] in_atomic(): 1, irqs_disabled(): 1, non_block: 0, pid: 0, name: swapper/1 [ 0.001257] preempt_count: 1, expected: 0 [ 0.001258] RCU nest depth: 0, expected: 0 [ 0.001259] Preemption disabled at: [ 0.001261] [<9000000000223800>] arch_dup_task_struct+0x20/0x110 [ 0.001272] CPU: 1 PID: 0 Comm: swapper/1 Not tainted 6.2.0-rc7+ #43 [ 0.001275] Hardware name: Loongson Loongson-3A5000-7A1000-1w-A2101/Loongson-LS3A5000-7A1000-1w-A2101, BIOS vUDK2018-LoongArch-V4.0.05132-beta10 12/13/202 [ 0.001277] Stack : 0072617764726148 0000000000000000 9000000000222f1c 90000001001e0000 [ 0.001286] 90000001001e3be0 90000001001e3be8 0000000000000000 0000000000000000 [ 0.001292] 90000001001e3be8 0000000000000040 90000001001e3cb8 90000001001e3a50 [ 0.001297] 9000000001642000 90000001001e3be8 be694d10ce4139dd 9000000100174500 [ 0.001303] 0000000000000001 0000000000000001 00000000ffffe0a2 0000000000000020 [ 0.001309] 000000000000002f 9000000001354116 00000000056b0000 ffffffffffffffff [ 0.001314] 0000000000000000 0000000000000000 90000000014f6e90 9000000001642000 [ 0.001320] 900000000022b69c 0000000000000001 0000000000000000 9000000001736a90 [ 0.001325] 9000000100038000 0000000000000000 9000000000222f34 0000000000000000 [ 0.001331] 00000000000000b0 0000000000000004 0000000000000000 0000000000070000 [ 0.001337] ... [ 0.001339] Call Trace: [ 0.001342] [<9000000000222f34>] show_stack+0x5c/0x180 [ 0.001346] [<90000000010bdd80>] dump_stack_lvl+0x60/0x88 [ 0.001352] [<9000000000266418>] __might_resched+0x180/0x1cc [ 0.001356] [<90000000010c742c>] mutex_lock+0x20/0x64 [ 0.001359] [<90000000002a8ccc>] irq_find_matching_fwspec+0x48/0x124 [ 0.001364] [<90000000002259c4>] constant_clockevent_init+0x68/0x204 [ 0.001368] [<900000000022acf4>] start_secondary+0x40/0xa8 [ 0.001371] [<90000000010c0124>] smpboot_entry+0x60/0x64 Here are the complete call chains: smpboot_entry() start_secondary() constant_clockevent_init() get_timer_irq() irq_find_matching_fwnode() irq_find_matching_fwspec() mutex_lock() might_sleep() __might_sleep() __might_resched() In order to avoid the above issue, we should break the call chains, using timer_irq_installed variable as check condition to only call get_timer_irq() once in constant_clockevent_init() is a simple and proper way. Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/kernel/time.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'arch/loongarch') diff --git a/arch/loongarch/kernel/time.c b/arch/loongarch/kernel/time.c index a6576dea590c..4351f69d9950 100644 --- a/arch/loongarch/kernel/time.c +++ b/arch/loongarch/kernel/time.c @@ -140,16 +140,17 @@ static int get_timer_irq(void) int constant_clockevent_init(void) { - int irq; unsigned int cpu = smp_processor_id(); unsigned long min_delta = 0x600; unsigned long max_delta = (1UL << 48) - 1; struct clock_event_device *cd; - static int timer_irq_installed = 0; + static int irq = 0, timer_irq_installed = 0; - irq = get_timer_irq(); - if (irq < 0) - pr_err("Failed to map irq %d (timer)\n", irq); + if (!timer_irq_installed) { + irq = get_timer_irq(); + if (irq < 0) + pr_err("Failed to map irq %d (timer)\n", irq); + } cd = &per_cpu(constant_clockevent_device, cpu); -- cgit v1.2.3 From 41596803302d83a67a80dc1efef4e51ac46acabb Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Sat, 25 Feb 2023 15:52:56 +0800 Subject: LoongArch: Make -mstrict-align configurable Introduce Kconfig option ARCH_STRICT_ALIGN to make -mstrict-align be configurable. Not all LoongArch cores support h/w unaligned access, we can use the -mstrict-align build parameter to prevent unaligned accesses. CPUs with h/w unaligned access support: Loongson-2K2000/2K3000/3A5000/3C5000/3D5000. CPUs without h/w unaligned access support: Loongson-2K500/2K1000. This option is enabled by default to make the kernel be able to run on all LoongArch systems. But you can disable it manually if you want to run kernel only on systems with h/w unaligned access support in order to optimise for performance. Reviewed-by: Arnd Bergmann Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 19 +++++++++++++++++++ arch/loongarch/Makefile | 5 +++++ arch/loongarch/kernel/Makefile | 4 +++- arch/loongarch/kernel/traps.c | 9 +++++++-- 4 files changed, 34 insertions(+), 3 deletions(-) (limited to 'arch/loongarch') diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 9cc8b84f7eb0..0c1c6063cc66 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -94,6 +94,7 @@ config LOONGARCH select HAVE_DYNAMIC_FTRACE_WITH_ARGS select HAVE_DYNAMIC_FTRACE_WITH_REGS select HAVE_EBPF_JIT + select HAVE_EFFICIENT_UNALIGNED_ACCESS if !ARCH_STRICT_ALIGN select HAVE_EXIT_THREAD select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD @@ -441,6 +442,24 @@ config ARCH_IOREMAP protection support. However, you can enable LoongArch DMW-based ioremap() for better performance. +config ARCH_STRICT_ALIGN + bool "Enable -mstrict-align to prevent unaligned accesses" if EXPERT + default y + help + Not all LoongArch cores support h/w unaligned access, we can use + -mstrict-align build parameter to prevent unaligned accesses. + + CPUs with h/w unaligned access support: + Loongson-2K2000/2K3000/3A5000/3C5000/3D5000. + + CPUs without h/w unaligned access support: + Loongson-2K500/2K1000. + + This option is enabled by default to make the kernel be able to run + on all LoongArch systems. But you can disable it manually if you want + to run kernel only on systems with h/w unaligned access support in + order to optimise for performance. + config KEXEC bool "Kexec system call" select KEXEC_CORE diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index 4402387d2755..6e1c931a8507 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -91,10 +91,15 @@ KBUILD_CPPFLAGS += -DVMLINUX_LOAD_ADDRESS=$(load-y) # instead of .eh_frame so we don't discard them. KBUILD_CFLAGS += -fno-asynchronous-unwind-tables +ifdef CONFIG_ARCH_STRICT_ALIGN # Don't emit unaligned accesses. # Not all LoongArch cores support unaligned access, and as kernel we can't # rely on others to provide emulation for these accesses. KBUILD_CFLAGS += $(call cc-option,-mstrict-align) +else +# Optimise for performance on hardware supports unaligned access. +KBUILD_CFLAGS += $(call cc-option,-mno-strict-align) +endif KBUILD_CFLAGS += -isystem $(shell $(CC) -print-file-name=include) diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index c8cfbd562921..df5dbabfe7a6 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -8,13 +8,15 @@ extra-y := vmlinux.lds obj-y += head.o cpu-probe.o cacheinfo.o env.o setup.o entry.o genex.o \ traps.o irq.o idle.o process.o dma.o mem.o io.o reset.o switch.o \ elf.o syscall.o signal.o time.o topology.o inst.o ptrace.o vdso.o \ - alternative.o unaligned.o unwind.o + alternative.o unwind.o obj-$(CONFIG_ACPI) += acpi.o obj-$(CONFIG_EFI) += efi.o obj-$(CONFIG_CPU_HAS_FPU) += fpu.o +obj-$(CONFIG_ARCH_STRICT_ALIGN) += unaligned.o + ifdef CONFIG_FUNCTION_TRACER ifndef CONFIG_DYNAMIC_FTRACE obj-y += mcount.o ftrace.o diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c index c38a146a973b..05511203732c 100644 --- a/arch/loongarch/kernel/traps.c +++ b/arch/loongarch/kernel/traps.c @@ -371,9 +371,14 @@ int no_unaligned_warning __read_mostly = 1; /* Only 1 warning by default */ asmlinkage void noinstr do_ale(struct pt_regs *regs) { - unsigned int *pc; irqentry_state_t state = irqentry_enter(regs); +#ifndef CONFIG_ARCH_STRICT_ALIGN + die_if_kernel("Kernel ale access", regs); + force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)regs->csr_badvaddr); +#else + unsigned int *pc; + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, regs->csr_badvaddr); /* @@ -397,8 +402,8 @@ asmlinkage void noinstr do_ale(struct pt_regs *regs) sigbus: die_if_kernel("Kernel ale access", regs); force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)regs->csr_badvaddr); - out: +#endif irqentry_exit(regs, state); } -- cgit v1.2.3 From f733f119e9b31088063652a7ad16963d85cb73dd Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Sat, 25 Feb 2023 15:52:56 +0800 Subject: LoongArch: Use la.pcrel instead of la.abs when it's trivially possible Let's start to kill la.abs in preparation for the subsequent support of the PIE kernel. BTW, Re-tab the indention in arch/loongarch/kernel/entry.S for alignment. Signed-off-by: Xi Ruoyao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/stackframe.h | 2 +- arch/loongarch/include/asm/uaccess.h | 1 - arch/loongarch/kernel/entry.S | 90 ++++++++++++++++----------------- arch/loongarch/kernel/head.S | 2 +- arch/loongarch/mm/tlbex.S | 3 +- 5 files changed, 48 insertions(+), 50 deletions(-) (limited to 'arch/loongarch') diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h index 4ca953062b5b..7deb043ce387 100644 --- a/arch/loongarch/include/asm/stackframe.h +++ b/arch/loongarch/include/asm/stackframe.h @@ -90,7 +90,7 @@ .endm .macro set_saved_sp stackp temp temp2 - la.abs \temp, kernelsp + la.pcrel \temp, kernelsp #ifdef CONFIG_SMP LONG_ADD \temp, \temp, u0 #endif diff --git a/arch/loongarch/include/asm/uaccess.h b/arch/loongarch/include/asm/uaccess.h index 255899d4a7c3..0d22991ae430 100644 --- a/arch/loongarch/include/asm/uaccess.h +++ b/arch/loongarch/include/asm/uaccess.h @@ -22,7 +22,6 @@ extern u64 __ua_limit; #define __UA_ADDR ".dword" -#define __UA_LA "la.abs" #define __UA_LIMIT __ua_limit /* diff --git a/arch/loongarch/kernel/entry.S b/arch/loongarch/kernel/entry.S index d53b631c9022..f14ab6a0e0b0 100644 --- a/arch/loongarch/kernel/entry.S +++ b/arch/loongarch/kernel/entry.S @@ -19,70 +19,70 @@ .cfi_sections .debug_frame .align 5 SYM_FUNC_START(handle_syscall) - csrrd t0, PERCPU_BASE_KS - la.abs t1, kernelsp - add.d t1, t1, t0 - move t2, sp - ld.d sp, t1, 0 + csrrd t0, PERCPU_BASE_KS + la.pcrel t1, kernelsp + add.d t1, t1, t0 + move t2, sp + ld.d sp, t1, 0 - addi.d sp, sp, -PT_SIZE - cfi_st t2, PT_R3 + addi.d sp, sp, -PT_SIZE + cfi_st t2, PT_R3 cfi_rel_offset sp, PT_R3 - st.d zero, sp, PT_R0 - csrrd t2, LOONGARCH_CSR_PRMD - st.d t2, sp, PT_PRMD - csrrd t2, LOONGARCH_CSR_CRMD - st.d t2, sp, PT_CRMD - csrrd t2, LOONGARCH_CSR_EUEN - st.d t2, sp, PT_EUEN - csrrd t2, LOONGARCH_CSR_ECFG - st.d t2, sp, PT_ECFG - csrrd t2, LOONGARCH_CSR_ESTAT - st.d t2, sp, PT_ESTAT - cfi_st ra, PT_R1 - cfi_st a0, PT_R4 - cfi_st a1, PT_R5 - cfi_st a2, PT_R6 - cfi_st a3, PT_R7 - cfi_st a4, PT_R8 - cfi_st a5, PT_R9 - cfi_st a6, PT_R10 - cfi_st a7, PT_R11 - csrrd ra, LOONGARCH_CSR_ERA - st.d ra, sp, PT_ERA + st.d zero, sp, PT_R0 + csrrd t2, LOONGARCH_CSR_PRMD + st.d t2, sp, PT_PRMD + csrrd t2, LOONGARCH_CSR_CRMD + st.d t2, sp, PT_CRMD + csrrd t2, LOONGARCH_CSR_EUEN + st.d t2, sp, PT_EUEN + csrrd t2, LOONGARCH_CSR_ECFG + st.d t2, sp, PT_ECFG + csrrd t2, LOONGARCH_CSR_ESTAT + st.d t2, sp, PT_ESTAT + cfi_st ra, PT_R1 + cfi_st a0, PT_R4 + cfi_st a1, PT_R5 + cfi_st a2, PT_R6 + cfi_st a3, PT_R7 + cfi_st a4, PT_R8 + cfi_st a5, PT_R9 + cfi_st a6, PT_R10 + cfi_st a7, PT_R11 + csrrd ra, LOONGARCH_CSR_ERA + st.d ra, sp, PT_ERA cfi_rel_offset ra, PT_ERA - cfi_st tp, PT_R2 - cfi_st u0, PT_R21 - cfi_st fp, PT_R22 + cfi_st tp, PT_R2 + cfi_st u0, PT_R21 + cfi_st fp, PT_R22 SAVE_STATIC - move u0, t0 - li.d tp, ~_THREAD_MASK - and tp, tp, sp + move u0, t0 + li.d tp, ~_THREAD_MASK + and tp, tp, sp - move a0, sp - bl do_syscall + move a0, sp + bl do_syscall RESTORE_ALL_AND_RET SYM_FUNC_END(handle_syscall) SYM_CODE_START(ret_from_fork) - bl schedule_tail # a0 = struct task_struct *prev - move a0, sp - bl syscall_exit_to_user_mode + bl schedule_tail # a0 = struct task_struct *prev + move a0, sp + bl syscall_exit_to_user_mode RESTORE_STATIC RESTORE_SOME RESTORE_SP_AND_RET SYM_CODE_END(ret_from_fork) SYM_CODE_START(ret_from_kernel_thread) - bl schedule_tail # a0 = struct task_struct *prev - move a0, s1 - jirl ra, s0, 0 - move a0, sp - bl syscall_exit_to_user_mode + bl schedule_tail # a0 = struct task_struct *prev + move a0, s1 + jirl ra, s0, 0 + move a0, sp + bl syscall_exit_to_user_mode RESTORE_STATIC RESTORE_SOME RESTORE_SP_AND_RET diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index 57bada6b4e93..aa6181714ec3 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -117,7 +117,7 @@ SYM_CODE_START(smpboot_entry) li.w t0, 0x00 # FPE=0, SXE=0, ASXE=0, BTE=0 csrwr t0, LOONGARCH_CSR_EUEN - la.abs t0, cpuboot_data + la.pcrel t0, cpuboot_data ld.d sp, t0, CPU_BOOT_STACK ld.d tp, t0, CPU_BOOT_TINFO diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S index 58781c6e4191..3dd2a9615cd9 100644 --- a/arch/loongarch/mm/tlbex.S +++ b/arch/loongarch/mm/tlbex.S @@ -24,8 +24,7 @@ move a0, sp REG_S a2, sp, PT_BVADDR li.w a1, \write - la.abs t0, do_page_fault - jirl ra, t0, 0 + bl do_page_fault RESTORE_ALL_AND_RET SYM_FUNC_END(tlb_do_page_fault_\write) .endm -- cgit v1.2.3 From 8cbd5ebfe241699288cb152081854414f6265718 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Sat, 25 Feb 2023 15:52:56 +0800 Subject: LoongArch: Add JUMP_VIRT_ADDR macro implementation to avoid using la.abs Add JUMP_VIRT_ADDR macro implementation to avoid using la.abs directly. This is a preparation for subsequent patches. Signed-off-by: Youling Tang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/stackframe.h | 9 +++++++++ arch/loongarch/kernel/head.S | 12 ++++-------- arch/loongarch/power/suspend_asm.S | 5 ++--- 3 files changed, 15 insertions(+), 11 deletions(-) (limited to 'arch/loongarch') diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h index 7deb043ce387..0274e704c13b 100644 --- a/arch/loongarch/include/asm/stackframe.h +++ b/arch/loongarch/include/asm/stackframe.h @@ -7,6 +7,7 @@ #include +#include #include #include #include @@ -36,6 +37,14 @@ cfi_restore \reg \offset \docfi .endm +/* Jump to the runtime virtual address. */ + .macro JUMP_VIRT_ADDR temp1 temp2 + li.d \temp1, CACHE_BASE + pcaddi \temp2, 0 + or \temp1, \temp1, \temp2 + jirl zero, \temp1, 0xc + .endm + .macro BACKUP_T0T1 csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index aa6181714ec3..d2ac26b5b22b 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -50,11 +50,8 @@ SYM_CODE_START(kernel_entry) # kernel entry point li.d t0, CSR_DMW1_INIT # CA, PLV0, 0x9000 xxxx xxxx xxxx csrwr t0, LOONGARCH_CSR_DMWIN1 - /* We might not get launched at the address the kernel is linked to, - so we jump there. */ - la.abs t0, 0f - jr t0 -0: + JUMP_VIRT_ADDR t0, t1 + /* Enable PG */ li.w t0, 0xb0 # PLV=0, IE=0, PG=1 csrwr t0, LOONGARCH_CSR_CRMD @@ -106,9 +103,8 @@ SYM_CODE_START(smpboot_entry) li.d t0, CSR_DMW1_INIT # CA, PLV0 csrwr t0, LOONGARCH_CSR_DMWIN1 - la.abs t0, 0f - jr t0 -0: + JUMP_VIRT_ADDR t0, t1 + /* Enable PG */ li.w t0, 0xb0 # PLV=0, IE=0, PG=1 csrwr t0, LOONGARCH_CSR_CRMD diff --git a/arch/loongarch/power/suspend_asm.S b/arch/loongarch/power/suspend_asm.S index eb2675642f9f..90da899c06a1 100644 --- a/arch/loongarch/power/suspend_asm.S +++ b/arch/loongarch/power/suspend_asm.S @@ -78,9 +78,8 @@ SYM_INNER_LABEL(loongarch_wakeup_start, SYM_L_GLOBAL) li.d t0, CSR_DMW1_INIT # CA, PLV0 csrwr t0, LOONGARCH_CSR_DMWIN1 - la.abs t0, 0f - jr t0 -0: + JUMP_VIRT_ADDR t0, t1 + la.pcrel t0, acpi_saved_sp ld.d sp, t0, 0 SETUP_WAKEUP -- cgit v1.2.3 From 396233c650084cb957eb6d87dd4abd3e56a7d499 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Sat, 25 Feb 2023 15:52:56 +0800 Subject: LoongArch: Add la_abs macro implementation Use the "la_abs macro" instead of the "la.abs pseudo instruction" to prepare for the subsequent PIE kernel. When PIE is not enabled, la_abs is equivalent to la.abs. Signed-off-by: Youling Tang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/asmmacro.h | 4 ++++ arch/loongarch/include/asm/stackframe.h | 2 +- arch/loongarch/kernel/genex.S | 8 ++++---- arch/loongarch/mm/tlbex.S | 14 +++++++------- 4 files changed, 16 insertions(+), 12 deletions(-) (limited to 'arch/loongarch') diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h index be037a40580d..cdc9935d5554 100644 --- a/arch/loongarch/include/asm/asmmacro.h +++ b/arch/loongarch/include/asm/asmmacro.h @@ -274,4 +274,8 @@ nor \dst, \src, zero .endm +.macro la_abs reg, sym + la.abs \reg, \sym +.endm + #endif /* _ASM_ASMMACRO_H */ diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h index 0274e704c13b..7df80e6ae9d2 100644 --- a/arch/loongarch/include/asm/stackframe.h +++ b/arch/loongarch/include/asm/stackframe.h @@ -86,7 +86,7 @@ * new value in sp. */ .macro get_saved_sp docfi=0 - la.abs t1, kernelsp + la_abs t1, kernelsp #ifdef CONFIG_SMP csrrd t0, PERCPU_BASE_KS LONG_ADD t1, t1, t0 diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S index 7e5c293ed89f..44ff1ff64260 100644 --- a/arch/loongarch/kernel/genex.S +++ b/arch/loongarch/kernel/genex.S @@ -34,7 +34,7 @@ SYM_FUNC_END(__arch_cpu_idle) SYM_FUNC_START(handle_vint) BACKUP_T0T1 SAVE_ALL - la.abs t1, __arch_cpu_idle + la_abs t1, __arch_cpu_idle LONG_L t0, sp, PT_ERA /* 32 byte rollback region */ ori t0, t0, 0x1f @@ -43,7 +43,7 @@ SYM_FUNC_START(handle_vint) LONG_S t0, sp, PT_ERA 1: move a0, sp move a1, sp - la.abs t0, do_vint + la_abs t0, do_vint jirl ra, t0, 0 RESTORE_ALL_AND_RET SYM_FUNC_END(handle_vint) @@ -72,7 +72,7 @@ SYM_FUNC_END(except_vec_cex) SAVE_ALL build_prep_\prep move a0, sp - la.abs t0, do_\handler + la_abs t0, do_\handler jirl ra, t0, 0 668: RESTORE_ALL_AND_RET @@ -93,6 +93,6 @@ SYM_FUNC_END(except_vec_cex) BUILD_HANDLER reserved reserved none /* others */ SYM_FUNC_START(handle_sys) - la.abs t0, handle_syscall + la_abs t0, handle_syscall jr t0 SYM_FUNC_END(handle_sys) diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S index 3dd2a9615cd9..244e2f5aeee5 100644 --- a/arch/loongarch/mm/tlbex.S +++ b/arch/loongarch/mm/tlbex.S @@ -39,7 +39,7 @@ SYM_FUNC_START(handle_tlb_protect) move a1, zero csrrd a2, LOONGARCH_CSR_BADV REG_S a2, sp, PT_BVADDR - la.abs t0, do_page_fault + la_abs t0, do_page_fault jirl ra, t0, 0 RESTORE_ALL_AND_RET SYM_FUNC_END(handle_tlb_protect) @@ -115,7 +115,7 @@ smp_pgtable_change_load: #ifdef CONFIG_64BIT vmalloc_load: - la.abs t1, swapper_pg_dir + la_abs t1, swapper_pg_dir b vmalloc_done_load #endif @@ -186,7 +186,7 @@ tlb_huge_update_load: nopage_tlb_load: dbar 0 csrrd ra, EXCEPTION_KS2 - la.abs t0, tlb_do_page_fault_0 + la_abs t0, tlb_do_page_fault_0 jr t0 SYM_FUNC_END(handle_tlb_load) @@ -262,7 +262,7 @@ smp_pgtable_change_store: #ifdef CONFIG_64BIT vmalloc_store: - la.abs t1, swapper_pg_dir + la_abs t1, swapper_pg_dir b vmalloc_done_store #endif @@ -335,7 +335,7 @@ tlb_huge_update_store: nopage_tlb_store: dbar 0 csrrd ra, EXCEPTION_KS2 - la.abs t0, tlb_do_page_fault_1 + la_abs t0, tlb_do_page_fault_1 jr t0 SYM_FUNC_END(handle_tlb_store) @@ -410,7 +410,7 @@ smp_pgtable_change_modify: #ifdef CONFIG_64BIT vmalloc_modify: - la.abs t1, swapper_pg_dir + la_abs t1, swapper_pg_dir b vmalloc_done_modify #endif @@ -482,7 +482,7 @@ tlb_huge_update_modify: nopage_tlb_modify: dbar 0 csrrd ra, EXCEPTION_KS2 - la.abs t0, tlb_do_page_fault_1 + la_abs t0, tlb_do_page_fault_1 jr t0 SYM_FUNC_END(handle_tlb_modify) -- cgit v1.2.3 From d8da19fbdedd5852592fbba18a7348e3f09500e6 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Sat, 25 Feb 2023 15:52:56 +0800 Subject: LoongArch: Add support for kernel relocation This config allows to compile kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR. Runtime relocation is possible since relocation metadata are embedded into the kernel. Signed-off-by: Youling Tang Signed-off-by: Xi Ruoyao # Use arch_initcall Signed-off-by: Jinyang He # Provide la_abs relocation code Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 8 +++ arch/loongarch/Makefile | 5 ++ arch/loongarch/include/asm/asmmacro.h | 13 +++++ arch/loongarch/include/asm/setup.h | 16 +++++ arch/loongarch/kernel/Makefile | 2 + arch/loongarch/kernel/head.S | 4 ++ arch/loongarch/kernel/relocate.c | 107 ++++++++++++++++++++++++++++++++++ arch/loongarch/kernel/vmlinux.lds.S | 20 ++++++- 8 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 arch/loongarch/kernel/relocate.c (limited to 'arch/loongarch') diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 0c1c6063cc66..32ab90dd76e5 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -493,6 +493,14 @@ config PHYSICAL_START specified in the "crashkernel=YM@XM" command line boot parameter passed to the panic-ed kernel). +config RELOCATABLE + bool "Relocatable kernel" + help + This builds the kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required, so as to relocate + the kernel binary at runtime to a different virtual address from + its link address. + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" depends on PROC_FS diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index 6e1c931a8507..768592998b39 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -71,6 +71,11 @@ KBUILD_AFLAGS_MODULE += -Wa,-mla-global-with-abs KBUILD_CFLAGS_MODULE += -fplt -Wa,-mla-global-with-abs,-mla-local-with-abs endif +ifeq ($(CONFIG_RELOCATABLE),y) +KBUILD_CFLAGS_KERNEL += -fPIE +LDFLAGS_vmlinux += -static -pie --no-dynamic-linker -z notext +endif + cflags-y += -ffreestanding cflags-y += $(call cc-option, -mno-check-zero-division) diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h index cdc9935d5554..c51a1b43acb4 100644 --- a/arch/loongarch/include/asm/asmmacro.h +++ b/arch/loongarch/include/asm/asmmacro.h @@ -275,7 +275,20 @@ .endm .macro la_abs reg, sym +#ifndef CONFIG_RELOCATABLE la.abs \reg, \sym +#else + 766: + lu12i.w \reg, 0 + ori \reg, \reg, 0 + lu32i.d \reg, 0 + lu52i.d \reg, \reg, 0 + .pushsection ".la_abs", "aw", %progbits + 768: + .dword 768b-766b + .dword \sym + .popsection +#endif .endm #endif /* _ASM_ASMMACRO_H */ diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h index 72ead58039f3..27d968655b4b 100644 --- a/arch/loongarch/include/asm/setup.h +++ b/arch/loongarch/include/asm/setup.h @@ -21,4 +21,20 @@ extern void per_cpu_trap_init(int cpu); extern void set_handler(unsigned long offset, void *addr, unsigned long len); extern void set_merr_handler(unsigned long offset, void *addr, unsigned long len); +#ifdef CONFIG_RELOCATABLE + +struct rela_la_abs { + long offset; + long symvalue; +}; + +extern long __la_abs_begin; +extern long __la_abs_end; +extern long __rela_dyn_begin; +extern long __rela_dyn_end; + +extern void __init relocate_kernel(void); + +#endif + #endif /* __SETUP_H */ diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index df5dbabfe7a6..94d33dbd4912 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -41,6 +41,8 @@ obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o +obj-$(CONFIG_RELOCATABLE) += relocate.o + obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index d2ac26b5b22b..c5c3ec2b819a 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -86,6 +86,10 @@ SYM_CODE_START(kernel_entry) # kernel entry point PTR_ADD sp, sp, tp set_saved_sp sp, t0, t1 +#ifdef CONFIG_RELOCATABLE + bl relocate_kernel +#endif + bl start_kernel ASM_BUG() diff --git a/arch/loongarch/kernel/relocate.c b/arch/loongarch/kernel/relocate.c new file mode 100644 index 000000000000..879c40372fba --- /dev/null +++ b/arch/loongarch/kernel/relocate.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for Kernel relocation at boot time + * + * Copyright (C) 2023 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +#define RELOCATED(x) ((void *)((long)x + reloc_offset)) + +static unsigned long reloc_offset; + +static inline void __init relocate_relative(void) +{ + Elf64_Rela *rela, *rela_end; + rela = (Elf64_Rela *)&__rela_dyn_begin; + rela_end = (Elf64_Rela *)&__rela_dyn_end; + + for ( ; rela < rela_end; rela++) { + Elf64_Addr addr = rela->r_offset; + Elf64_Addr relocated_addr = rela->r_addend; + + if (rela->r_info != R_LARCH_RELATIVE) + continue; + + if (relocated_addr >= VMLINUX_LOAD_ADDRESS) + relocated_addr = (Elf64_Addr)RELOCATED(relocated_addr); + + *(Elf64_Addr *)RELOCATED(addr) = relocated_addr; + } +} + +static inline void __init relocate_absolute(void) +{ + void *begin, *end; + struct rela_la_abs *p; + + begin = &__la_abs_begin; + end = &__la_abs_end; + + for (p = begin; (void *)p < end; p++) { + long v = p->symvalue; + uint32_t lu12iw, ori, lu32id, lu52id; + union loongarch_instruction *insn = (void *)p - p->offset; + + lu12iw = (v >> 12) & 0xfffff; + ori = v & 0xfff; + lu32id = (v >> 32) & 0xfffff; + lu52id = v >> 52; + + insn[0].reg1i20_format.immediate = lu12iw; + insn[1].reg2i12_format.immediate = ori; + insn[2].reg1i20_format.immediate = lu32id; + insn[3].reg2i12_format.immediate = lu52id; + } +} + +void __init relocate_kernel(void) +{ + reloc_offset = (unsigned long)_text - VMLINUX_LOAD_ADDRESS; + + if (reloc_offset) + relocate_relative(); + + relocate_absolute(); +} + +/* + * Show relocation information on panic. + */ +static void show_kernel_relocation(const char *level) +{ + if (reloc_offset > 0) { + printk(level); + pr_cont("Kernel relocated by 0x%lx\n", reloc_offset); + pr_cont(" .text @ 0x%px\n", _text); + pr_cont(" .data @ 0x%px\n", _sdata); + pr_cont(" .bss @ 0x%px\n", __bss_start); + } +} + +static int kernel_location_notifier_fn(struct notifier_block *self, + unsigned long v, void *p) +{ + show_kernel_relocation(KERN_EMERG); + return NOTIFY_DONE; +} + +static struct notifier_block kernel_location_notifier = { + .notifier_call = kernel_location_notifier_fn +}; + +static int __init register_kernel_offset_dumper(void) +{ + atomic_notifier_chain_register(&panic_notifier_list, + &kernel_location_notifier); + return 0; +} + +arch_initcall(register_kernel_offset_dumper); diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S index 733b16e8d55d..ad2ce1a0908e 100644 --- a/arch/loongarch/kernel/vmlinux.lds.S +++ b/arch/loongarch/kernel/vmlinux.lds.S @@ -66,10 +66,21 @@ SECTIONS __alt_instructions_end = .; } +#ifdef CONFIG_RELOCATABLE + . = ALIGN(8); + .la_abs : AT(ADDR(.la_abs) - LOAD_OFFSET) { + __la_abs_begin = .; + *(.la_abs) + __la_abs_end = .; + } +#endif + .got : ALIGN(16) { *(.got) } .plt : ALIGN(16) { *(.plt) } .got.plt : ALIGN(16) { *(.got.plt) } + .data.rel : { *(.data.rel*) } + . = ALIGN(PECOFF_SEGMENT_ALIGN); __init_begin = .; __inittext_begin = .; @@ -93,8 +104,6 @@ SECTIONS PERCPU_SECTION(1 << CONFIG_L1_CACHE_SHIFT) #endif - .rela.dyn : ALIGN(8) { *(.rela.dyn) *(.rela*) } - .init.bss : { *(.init.bss) } @@ -107,6 +116,12 @@ SECTIONS RO_DATA(4096) RW_DATA(1 << CONFIG_L1_CACHE_SHIFT, PAGE_SIZE, THREAD_SIZE) + .rela.dyn : ALIGN(8) { + __rela_dyn_begin = .; + *(.rela.dyn) *(.rela*) + __rela_dyn_end = .; + } + .sdata : { *(.sdata) } @@ -133,6 +148,7 @@ SECTIONS DISCARDS /DISCARD/ : { + *(.dynamic .dynsym .dynstr .hash .gnu.hash) *(.gnu.attributes) *(.options) *(.eh_frame) -- cgit v1.2.3 From e5f02b51fa0cb785e352e77271a65e96051b789b Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Sat, 25 Feb 2023 15:52:56 +0800 Subject: LoongArch: Add support for kernel address space layout randomization (KASLR) This patch adds support for relocating the kernel to a random address. Entropy is derived from the banner, which will change every build and random_get_entropy() which should provide additional runtime entropy. The kernel is relocated by up to RANDOMIZE_BASE_MAX_OFFSET bytes from its link address. Because relocation happens so early during the kernel booting, the amount of physical memory has not yet been determined. This means the only way to limit relocation within the available memory is via Kconfig. So we limit the maximum value of RANDOMIZE_BASE_MAX_OFFSET to 256M (0x10000000) because our memory layout has many holes. Signed-off-by: Youling Tang Signed-off-by: Xi Ruoyao # Fix compiler warnings Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 25 +++++++ arch/loongarch/include/asm/setup.h | 2 +- arch/loongarch/kernel/head.S | 13 ++++ arch/loongarch/kernel/relocate.c | 145 +++++++++++++++++++++++++++++++++++-- 4 files changed, 179 insertions(+), 6 deletions(-) (limited to 'arch/loongarch') diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 32ab90dd76e5..2fc18a3c565e 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -501,6 +501,31 @@ config RELOCATABLE the kernel binary at runtime to a different virtual address from its link address. +config RANDOMIZE_BASE + bool "Randomize the address of the kernel (KASLR)" + depends on RELOCATABLE + help + Randomizes the physical and virtual address at which the + kernel image is loaded, as a security feature that + deters exploit attempts relying on knowledge of the location + of kernel internals. + + The kernel will be offset by up to RANDOMIZE_BASE_MAX_OFFSET. + + If unsure, say N. + +config RANDOMIZE_BASE_MAX_OFFSET + hex "Maximum KASLR offset" if EXPERT + depends on RANDOMIZE_BASE + range 0x0 0x10000000 + default "0x01000000" + help + When KASLR is active, this provides the maximum offset that will + be applied to the kernel image. It should be set according to the + amount of physical RAM available in the target system. + + This is limited by the size of the lower address memory, 256MB. + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" depends on PROC_FS diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h index 27d968655b4b..be05c0e706a2 100644 --- a/arch/loongarch/include/asm/setup.h +++ b/arch/loongarch/include/asm/setup.h @@ -33,7 +33,7 @@ extern long __la_abs_end; extern long __rela_dyn_begin; extern long __rela_dyn_end; -extern void __init relocate_kernel(void); +extern void * __init relocate_kernel(void); #endif diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index c5c3ec2b819a..1d35becc01ee 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -87,7 +87,20 @@ SYM_CODE_START(kernel_entry) # kernel entry point set_saved_sp sp, t0, t1 #ifdef CONFIG_RELOCATABLE + bl relocate_kernel + +#ifdef CONFIG_RANDOMIZE_BASE + /* Repoint the sp into the new kernel */ + PTR_LI sp, (_THREAD_SIZE - PT_SIZE) + PTR_ADD sp, sp, tp + set_saved_sp sp, t0, t1 +#endif + + /* relocate_kernel() returns the new kernel entry point */ + jr a0 + ASM_BUG() + #endif bl start_kernel diff --git a/arch/loongarch/kernel/relocate.c b/arch/loongarch/kernel/relocate.c index 879c40372fba..01f94d1e3edf 100644 --- a/arch/loongarch/kernel/relocate.c +++ b/arch/loongarch/kernel/relocate.c @@ -9,11 +9,15 @@ #include #include #include +#include +#include +#include #include #include #include #define RELOCATED(x) ((void *)((long)x + reloc_offset)) +#define RELOCATED_KASLR(x) ((void *)((long)x + random_offset)) static unsigned long reloc_offset; @@ -37,13 +41,13 @@ static inline void __init relocate_relative(void) } } -static inline void __init relocate_absolute(void) +static inline void __init relocate_absolute(long random_offset) { void *begin, *end; struct rela_la_abs *p; - begin = &__la_abs_begin; - end = &__la_abs_end; + begin = RELOCATED_KASLR(&__la_abs_begin); + end = RELOCATED_KASLR(&__la_abs_end); for (p = begin; (void *)p < end; p++) { long v = p->symvalue; @@ -62,14 +66,145 @@ static inline void __init relocate_absolute(void) } } -void __init relocate_kernel(void) +#ifdef CONFIG_RANDOMIZE_BASE +static inline __init unsigned long rotate_xor(unsigned long hash, + const void *area, size_t size) { + size_t i, diff; + const typeof(hash) *ptr = PTR_ALIGN(area, sizeof(hash)); + + diff = (void *)ptr - area; + if (size < diff + sizeof(hash)) + return hash; + + size = ALIGN_DOWN(size - diff, sizeof(hash)); + + for (i = 0; i < size / sizeof(hash); i++) { + /* Rotate by odd number of bits and XOR. */ + hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7); + hash ^= ptr[i]; + } + + return hash; +} + +static inline __init unsigned long get_random_boot(void) +{ + unsigned long hash = 0; + unsigned long entropy = random_get_entropy(); + + /* Attempt to create a simple but unpredictable starting entropy. */ + hash = rotate_xor(hash, linux_banner, strlen(linux_banner)); + + /* Add in any runtime entropy we can get */ + hash = rotate_xor(hash, &entropy, sizeof(entropy)); + + return hash; +} + +static inline __init bool kaslr_disabled(void) +{ + char *str; + const char *builtin_cmdline = CONFIG_CMDLINE; + + str = strstr(builtin_cmdline, "nokaslr"); + if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' ')) + return true; + + str = strstr(boot_command_line, "nokaslr"); + if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) + return true; + + return false; +} + +/* Choose a new address for the kernel */ +static inline void __init *determine_relocation_address(void) +{ + unsigned long kernel_length; + unsigned long random_offset; + void *destination = _text; + + if (kaslr_disabled()) + return destination; + + kernel_length = (long)_end - (long)_text; + + random_offset = get_random_boot() << 16; + random_offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1); + if (random_offset < kernel_length) + random_offset += ALIGN(kernel_length, 0xffff); + + return RELOCATED_KASLR(destination); +} + +static inline int __init relocation_addr_valid(void *location_new) +{ + if ((unsigned long)location_new & 0x00000ffff) + return 0; /* Inappropriately aligned new location */ + + if ((unsigned long)location_new < (unsigned long)_end) + return 0; /* New location overlaps original kernel */ + + return 1; +} +#endif + +static inline void __init update_reloc_offset(unsigned long *addr, long random_offset) +{ + unsigned long *new_addr = (unsigned long *)RELOCATED_KASLR(addr); + + *new_addr = (unsigned long)reloc_offset; +} + +void * __init relocate_kernel(void) +{ + unsigned long kernel_length; + unsigned long random_offset = 0; + void *location_new = _text; /* Default to original kernel start */ + void *kernel_entry = start_kernel; /* Default to original kernel entry point */ + char *cmdline = early_ioremap(fw_arg1, COMMAND_LINE_SIZE); /* Boot command line is passed in fw_arg1 */ + + strscpy(boot_command_line, cmdline, COMMAND_LINE_SIZE); + +#ifdef CONFIG_RANDOMIZE_BASE + location_new = determine_relocation_address(); + + /* Sanity check relocation address */ + if (relocation_addr_valid(location_new)) + random_offset = (unsigned long)location_new - (unsigned long)(_text); +#endif reloc_offset = (unsigned long)_text - VMLINUX_LOAD_ADDRESS; + if (random_offset) { + kernel_length = (long)(_end) - (long)(_text); + + /* Copy the kernel to it's new location */ + memcpy(location_new, _text, kernel_length); + + /* Sync the caches ready for execution of new kernel */ + __asm__ __volatile__ ( + "ibar 0 \t\n" + "dbar 0 \t\n" + ::: "memory"); + + reloc_offset += random_offset; + + /* Return the new kernel's entry point */ + kernel_entry = RELOCATED_KASLR(start_kernel); + + /* The current thread is now within the relocated kernel */ + __current_thread_info = RELOCATED_KASLR(__current_thread_info); + + update_reloc_offset(&reloc_offset, random_offset); + } + if (reloc_offset) relocate_relative(); - relocate_absolute(); + relocate_absolute(random_offset); + + return kernel_entry; } /* -- cgit v1.2.3 From 3f89765d6227fe4645a91e062332b6b4eb24072c Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Sat, 25 Feb 2023 15:52:56 +0800 Subject: LoongArch: kdump: Add single kernel image implementation This feature depends on the kernel being relocatable. Enable using single kernel image for kdump, and then no longer need to build two kernels (production kernel and capture kernel share a single kernel image). Also enable CONFIG_CRASH_DUMP in loongson3_defconfig. Signed-off-by: Youling Tang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 12 +----------- arch/loongarch/Makefile | 4 ---- arch/loongarch/configs/loongson3_defconfig | 1 + arch/loongarch/include/asm/addrspace.h | 2 ++ arch/loongarch/kernel/head.S | 2 +- 5 files changed, 5 insertions(+), 16 deletions(-) (limited to 'arch/loongarch') diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 2fc18a3c565e..c6a02d26b3e5 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -473,6 +473,7 @@ config KEXEC config CRASH_DUMP bool "Build kdump crash kernel" + select RELOCATABLE help Generate crash dump after being started by kexec. This should be normally only set in special crash dump kernels which are @@ -482,17 +483,6 @@ config CRASH_DUMP For more details see Documentation/admin-guide/kdump/kdump.rst -config PHYSICAL_START - hex "Physical address where the kernel is loaded" - default "0x90000000a0000000" - depends on CRASH_DUMP - help - This gives the XKPRANGE address where the kernel is loaded. - If you plan to use kernel for capturing the crash dump change - this value to start of the reserved region (the "X" value as - specified in the "crashkernel=YM@XM" command line boot parameter - passed to the panic-ed kernel). - config RELOCATABLE bool "Relocatable kernel" help diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index 768592998b39..f71edf574101 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -79,11 +79,7 @@ endif cflags-y += -ffreestanding cflags-y += $(call cc-option, -mno-check-zero-division) -ifndef CONFIG_PHYSICAL_START load-y = 0x9000000000200000 -else -load-y = $(CONFIG_PHYSICAL_START) -endif bootvars-y = VMLINUX_LOAD_ADDRESS=$(load-y) drivers-$(CONFIG_PCI) += arch/loongarch/pci/ diff --git a/arch/loongarch/configs/loongson3_defconfig b/arch/loongarch/configs/loongson3_defconfig index eb84cae642e5..e18213f01cc4 100644 --- a/arch/loongarch/configs/loongson3_defconfig +++ b/arch/loongarch/configs/loongson3_defconfig @@ -48,6 +48,7 @@ CONFIG_HOTPLUG_CPU=y CONFIG_NR_CPUS=64 CONFIG_NUMA=y CONFIG_KEXEC=y +CONFIG_CRASH_DUMP=y CONFIG_SUSPEND=y CONFIG_HIBERNATION=y CONFIG_ACPI=y diff --git a/arch/loongarch/include/asm/addrspace.h b/arch/loongarch/include/asm/addrspace.h index d342935e5a72..8fb699b4d40a 100644 --- a/arch/loongarch/include/asm/addrspace.h +++ b/arch/loongarch/include/asm/addrspace.h @@ -125,4 +125,6 @@ extern unsigned long vm_map_base; #define ISA_IOSIZE SZ_16K #define IO_SPACE_LIMIT (PCI_IOSIZE - 1) +#define PHYS_LINK_KADDR PHYSADDR(VMLINUX_LOAD_ADDRESS) + #endif /* _ASM_ADDRSPACE_H */ diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index 1d35becc01ee..aa64b179744f 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -24,7 +24,7 @@ _head: .org 0x8 .dword kernel_entry /* Kernel entry point */ .dword _end - _text /* Kernel image effective size */ - .quad 0 /* Kernel image load offset from start of RAM */ + .quad PHYS_LINK_KADDR /* Kernel image load offset from start of RAM */ .org 0x38 /* 0x20 ~ 0x37 reserved */ .long LINUX_PE_MAGIC .long pe_header - _head /* Offset to the PE header */ -- cgit v1.2.3 From 35c94fab6eee72d60d9cd8d1e6e43e1f77b1dce2 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Sat, 25 Feb 2023 15:52:56 +0800 Subject: LoongArch: kdump: Add crashkernel=YM handling When the kernel crashkernel parameter is specified with just a size, we are supposed to allocate a region from RAM to store the crashkernel, "crashkernel=512M" would be recommended for kdump. Fix this by lifting similar code from x86, importing it to LoongArch with LoongArch specific parameters added. We allocate the crashkernel region from the first 4GB of physical memory (because SWIOTLB should be allocated below 4GB). However, LoongArch currently does not implement crashkernel_low and crashkernel_high the same as x86. When X is not specified, crash_base defaults to 0 (crashkernel=YM@XM). Signed-off-by: Youling Tang Signed-off-by: Huacai Chen --- arch/loongarch/kernel/setup.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'arch/loongarch') diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 4344502c0b31..bae84ccf6d36 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -234,11 +234,14 @@ static void __init arch_reserve_vmcore(void) #endif } +/* 2MB alignment for crash kernel regions */ +#define CRASH_ALIGN SZ_2M +#define CRASH_ADDR_MAX SZ_4G + static void __init arch_parse_crashkernel(void) { #ifdef CONFIG_KEXEC int ret; - unsigned long long start; unsigned long long total_mem; unsigned long long crash_base, crash_size; @@ -247,8 +250,13 @@ static void __init arch_parse_crashkernel(void) if (ret < 0 || crash_size <= 0) return; - start = memblock_phys_alloc_range(crash_size, 1, crash_base, crash_base + crash_size); - if (start != crash_base) { + if (crash_base <= 0) { + crash_base = memblock_phys_alloc_range(crash_size, CRASH_ALIGN, CRASH_ALIGN, CRASH_ADDR_MAX); + if (!crash_base) { + pr_warn("crashkernel reservation failed - No suitable area found.\n"); + return; + } + } else if (!memblock_phys_alloc_range(crash_size, CRASH_ALIGN, crash_base, crash_base + crash_size)) { pr_warn("Invalid memory region reserved for crash kernel\n"); return; } -- cgit v1.2.3 From edffa33c7bb5a73e90c754c7a497162b77d7c55f Mon Sep 17 00:00:00 2001 From: Qing Zhang Date: Sat, 25 Feb 2023 15:52:57 +0800 Subject: LoongArch: Add hardware breakpoints/watchpoints support Use perf framework to manage hardware instruction and data breakpoints. LoongArch defines hardware watchpoint functions for instruction fetch and memory load/store operations. After the software configures hardware watchpoints, the processor hardware will monitor the access address of the instruction fetch and load/store operation, and trigger an exception of the watchpoint when it meets the conditions set by the watchpoint. The hardware monitoring points for instruction fetching and load/store operations each have a register for the overall configuration of all monitoring points, a register for recording the status of all monitoring points, and four registers required for configuration of each watchpoint individually. Signed-off-by: Qing Zhang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/hw_breakpoint.h | 145 ++++++++ arch/loongarch/include/asm/loongarch.h | 32 +- arch/loongarch/include/asm/processor.h | 15 +- arch/loongarch/include/asm/switch_to.h | 1 + arch/loongarch/kernel/Makefile | 1 + arch/loongarch/kernel/hw_breakpoint.c | 523 +++++++++++++++++++++++++++++ arch/loongarch/kernel/process.c | 7 + arch/loongarch/kernel/traps.c | 10 + 9 files changed, 713 insertions(+), 22 deletions(-) create mode 100644 arch/loongarch/include/asm/hw_breakpoint.h create mode 100644 arch/loongarch/kernel/hw_breakpoint.c (limited to 'arch/loongarch') diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index c6a02d26b3e5..97423f86384b 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -101,6 +101,7 @@ config LOONGARCH select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER select HAVE_GENERIC_VDSO + select HAVE_HW_BREAKPOINT if PERF_EVENTS select HAVE_IOREMAP_PROT select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_IRQ_TIME_ACCOUNTING diff --git a/arch/loongarch/include/asm/hw_breakpoint.h b/arch/loongarch/include/asm/hw_breakpoint.h new file mode 100644 index 000000000000..21447fb1efc7 --- /dev/null +++ b/arch/loongarch/include/asm/hw_breakpoint.h @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022-2023 Loongson Technology Corporation Limited + */ +#ifndef __ASM_HW_BREAKPOINT_H +#define __ASM_HW_BREAKPOINT_H + +#include + +#ifdef __KERNEL__ + +/* Breakpoint */ +#define LOONGARCH_BREAKPOINT_EXECUTE (0 << 0) + +/* Watchpoints */ +#define LOONGARCH_BREAKPOINT_LOAD (1 << 0) +#define LOONGARCH_BREAKPOINT_STORE (1 << 1) + +struct arch_hw_breakpoint_ctrl { + u32 __reserved : 28, + len : 2, + type : 2; +}; + +struct arch_hw_breakpoint { + u64 address; + u64 mask; + struct arch_hw_breakpoint_ctrl ctrl; +}; + +/* Lengths */ +#define LOONGARCH_BREAKPOINT_LEN_1 0b11 +#define LOONGARCH_BREAKPOINT_LEN_2 0b10 +#define LOONGARCH_BREAKPOINT_LEN_4 0b01 +#define LOONGARCH_BREAKPOINT_LEN_8 0b00 + +/* + * Limits. + * Changing these will require modifications to the register accessors. + */ +#define LOONGARCH_MAX_BRP 8 +#define LOONGARCH_MAX_WRP 8 + +/* Virtual debug register bases. */ +#define CSR_CFG_ADDR 0 +#define CSR_CFG_MASK (CSR_CFG_ADDR + LOONGARCH_MAX_BRP) +#define CSR_CFG_CTRL (CSR_CFG_MASK + LOONGARCH_MAX_BRP) +#define CSR_CFG_ASID (CSR_CFG_CTRL + LOONGARCH_MAX_WRP) + +/* Debug register names. */ +#define LOONGARCH_CSR_NAME_ADDR ADDR +#define LOONGARCH_CSR_NAME_MASK MASK +#define LOONGARCH_CSR_NAME_CTRL CTRL +#define LOONGARCH_CSR_NAME_ASID ASID + +/* Accessor macros for the debug registers. */ +#define LOONGARCH_CSR_WATCH_READ(N, REG, T, VAL) \ +do { \ + if (T == 0) \ + VAL = csr_read64(LOONGARCH_CSR_##IB##N##REG); \ + else \ + VAL = csr_read64(LOONGARCH_CSR_##DB##N##REG); \ +} while (0) + +#define LOONGARCH_CSR_WATCH_WRITE(N, REG, T, VAL) \ +do { \ + if (T == 0) \ + csr_write64(VAL, LOONGARCH_CSR_##IB##N##REG); \ + else \ + csr_write64(VAL, LOONGARCH_CSR_##DB##N##REG); \ +} while (0) + +/* Exact number */ +#define CSR_FWPC_NUM 0x3f +#define CSR_MWPC_NUM 0x3f + +#define CTRL_PLV_ENABLE 0x1e + +#define MWPnCFG3_LoadEn 8 +#define MWPnCFG3_StoreEn 9 + +#define MWPnCFG3_Type_mask 0x3 +#define MWPnCFG3_Size_mask 0x3 + +static inline u32 encode_ctrl_reg(struct arch_hw_breakpoint_ctrl ctrl) +{ + return (ctrl.len << 10) | (ctrl.type << 8); +} + +static inline void decode_ctrl_reg(u32 reg, struct arch_hw_breakpoint_ctrl *ctrl) +{ + reg >>= 8; + ctrl->type = reg & MWPnCFG3_Type_mask; + reg >>= 2; + ctrl->len = reg & MWPnCFG3_Size_mask; +} + +struct task_struct; +struct notifier_block; +struct perf_event; +struct perf_event_attr; + +extern int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, + int *gen_len, int *gen_type, int *offset); +extern int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw); +extern int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw); +extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused, + unsigned long val, void *data); + +extern int arch_install_hw_breakpoint(struct perf_event *bp); +extern void arch_uninstall_hw_breakpoint(struct perf_event *bp); +extern int hw_breakpoint_slots(int type); +extern void hw_breakpoint_pmu_read(struct perf_event *bp); + +void breakpoint_handler(struct pt_regs *regs); +void watchpoint_handler(struct pt_regs *regs); + +#ifdef CONFIG_HAVE_HW_BREAKPOINT +extern void ptrace_hw_copy_thread(struct task_struct *task); +extern void hw_breakpoint_thread_switch(struct task_struct *next); +#else +static inline void ptrace_hw_copy_thread(struct task_struct *task) +{ +} +static inline void hw_breakpoint_thread_switch(struct task_struct *next) +{ +} +#endif + +/* Determine number of BRP registers available. */ +static inline int get_num_brps(void) +{ + return csr_read64(LOONGARCH_CSR_FWPC) & CSR_FWPC_NUM; +} + +/* Determine number of WRP registers available. */ +static inline int get_num_wrps(void) +{ + return csr_read64(LOONGARCH_CSR_MWPC) & CSR_MWPC_NUM; +} + +#endif /* __KERNEL__ */ +#endif /* __ASM_BREAKPOINT_H */ diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index 7f8d57a61c8b..e9aed583a064 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -970,42 +970,42 @@ static __always_inline void iocsr_write64(u64 val, u32 reg) #define LOONGARCH_CSR_DB0ADDR 0x310 /* data breakpoint 0 address */ #define LOONGARCH_CSR_DB0MASK 0x311 /* data breakpoint 0 mask */ -#define LOONGARCH_CSR_DB0CTL 0x312 /* data breakpoint 0 control */ +#define LOONGARCH_CSR_DB0CTRL 0x312 /* data breakpoint 0 control */ #define LOONGARCH_CSR_DB0ASID 0x313 /* data breakpoint 0 asid */ #define LOONGARCH_CSR_DB1ADDR 0x318 /* data breakpoint 1 address */ #define LOONGARCH_CSR_DB1MASK 0x319 /* data breakpoint 1 mask */ -#define LOONGARCH_CSR_DB1CTL 0x31a /* data breakpoint 1 control */ +#define LOONGARCH_CSR_DB1CTRL 0x31a /* data breakpoint 1 control */ #define LOONGARCH_CSR_DB1ASID 0x31b /* data breakpoint 1 asid */ #define LOONGARCH_CSR_DB2ADDR 0x320 /* data breakpoint 2 address */ #define LOONGARCH_CSR_DB2MASK 0x321 /* data breakpoint 2 mask */ -#define LOONGARCH_CSR_DB2CTL 0x322 /* data breakpoint 2 control */ +#define LOONGARCH_CSR_DB2CTRL 0x322 /* data breakpoint 2 control */ #define LOONGARCH_CSR_DB2ASID 0x323 /* data breakpoint 2 asid */ #define LOONGARCH_CSR_DB3ADDR 0x328 /* data breakpoint 3 address */ #define LOONGARCH_CSR_DB3MASK 0x329 /* data breakpoint 3 mask */ -#define LOONGARCH_CSR_DB3CTL 0x32a /* data breakpoint 3 control */ +#define LOONGARCH_CSR_DB3CTRL 0x32a /* data breakpoint 3 control */ #define LOONGARCH_CSR_DB3ASID 0x32b /* data breakpoint 3 asid */ #define LOONGARCH_CSR_DB4ADDR 0x330 /* data breakpoint 4 address */ #define LOONGARCH_CSR_DB4MASK 0x331 /* data breakpoint 4 maks */ -#define LOONGARCH_CSR_DB4CTL 0x332 /* data breakpoint 4 control */ +#define LOONGARCH_CSR_DB4CTRL 0x332 /* data breakpoint 4 control */ #define LOONGARCH_CSR_DB4ASID 0x333 /* data breakpoint 4 asid */ #define LOONGARCH_CSR_DB5ADDR 0x338 /* data breakpoint 5 address */ #define LOONGARCH_CSR_DB5MASK 0x339 /* data breakpoint 5 mask */ -#define LOONGARCH_CSR_DB5CTL 0x33a /* data breakpoint 5 control */ +#define LOONGARCH_CSR_DB5CTRL 0x33a /* data breakpoint 5 control */ #define LOONGARCH_CSR_DB5ASID 0x33b /* data breakpoint 5 asid */ #define LOONGARCH_CSR_DB6ADDR 0x340 /* data breakpoint 6 address */ #define LOONGARCH_CSR_DB6MASK 0x341 /* data breakpoint 6 mask */ -#define LOONGARCH_CSR_DB6CTL 0x342 /* data breakpoint 6 control */ +#define LOONGARCH_CSR_DB6CTRL 0x342 /* data breakpoint 6 control */ #define LOONGARCH_CSR_DB6ASID 0x343 /* data breakpoint 6 asid */ #define LOONGARCH_CSR_DB7ADDR 0x348 /* data breakpoint 7 address */ #define LOONGARCH_CSR_DB7MASK 0x349 /* data breakpoint 7 mask */ -#define LOONGARCH_CSR_DB7CTL 0x34a /* data breakpoint 7 control */ +#define LOONGARCH_CSR_DB7CTRL 0x34a /* data breakpoint 7 control */ #define LOONGARCH_CSR_DB7ASID 0x34b /* data breakpoint 7 asid */ #define LOONGARCH_CSR_FWPC 0x380 /* instruction breakpoint config */ @@ -1013,42 +1013,42 @@ static __always_inline void iocsr_write64(u64 val, u32 reg) #define LOONGARCH_CSR_IB0ADDR 0x390 /* inst breakpoint 0 address */ #define LOONGARCH_CSR_IB0MASK 0x391 /* inst breakpoint 0 mask */ -#define LOONGARCH_CSR_IB0CTL 0x392 /* inst breakpoint 0 control */ +#define LOONGARCH_CSR_IB0CTRL 0x392 /* inst breakpoint 0 control */ #define LOONGARCH_CSR_IB0ASID 0x393 /* inst breakpoint 0 asid */ #define LOONGARCH_CSR_IB1ADDR 0x398 /* inst breakpoint 1 address */ #define LOONGARCH_CSR_IB1MASK 0x399 /* inst breakpoint 1 mask */ -#define LOONGARCH_CSR_IB1CTL 0x39a /* inst breakpoint 1 control */ +#define LOONGARCH_CSR_IB1CTRL 0x39a /* inst breakpoint 1 control */ #define LOONGARCH_CSR_IB1ASID 0x39b /* inst breakpoint 1 asid */ #define LOONGARCH_CSR_IB2ADDR 0x3a0 /* inst breakpoint 2 address */ #define LOONGARCH_CSR_IB2MASK 0x3a1 /* inst breakpoint 2 mask */ -#define LOONGARCH_CSR_IB2CTL 0x3a2 /* inst breakpoint 2 control */ +#define LOONGARCH_CSR_IB2CTRL 0x3a2 /* inst breakpoint 2 control */ #define LOONGARCH_CSR_IB2ASID 0x3a3 /* inst breakpoint 2 asid */ #define LOONGARCH_CSR_IB3ADDR 0x3a8 /* inst breakpoint 3 address */ #define LOONGARCH_CSR_IB3MASK 0x3a9 /* breakpoint 3 mask */ -#define LOONGARCH_CSR_IB3CTL 0x3aa /* inst breakpoint 3 control */ +#define LOONGARCH_CSR_IB3CTRL 0x3aa /* inst breakpoint 3 control */ #define LOONGARCH_CSR_IB3ASID 0x3ab /* inst breakpoint 3 asid */ #define LOONGARCH_CSR_IB4ADDR 0x3b0 /* inst breakpoint 4 address */ #define LOONGARCH_CSR_IB4MASK 0x3b1 /* inst breakpoint 4 mask */ -#define LOONGARCH_CSR_IB4CTL 0x3b2 /* inst breakpoint 4 control */ +#define LOONGARCH_CSR_IB4CTRL 0x3b2 /* inst breakpoint 4 control */ #define LOONGARCH_CSR_IB4ASID 0x3b3 /* inst breakpoint 4 asid */ #define LOONGARCH_CSR_IB5ADDR 0x3b8 /* inst breakpoint 5 address */ #define LOONGARCH_CSR_IB5MASK 0x3b9 /* inst breakpoint 5 mask */ -#define LOONGARCH_CSR_IB5CTL 0x3ba /* inst breakpoint 5 control */ +#define LOONGARCH_CSR_IB5CTRL 0x3ba /* inst breakpoint 5 control */ #define LOONGARCH_CSR_IB5ASID 0x3bb /* inst breakpoint 5 asid */ #define LOONGARCH_CSR_IB6ADDR 0x3c0 /* inst breakpoint 6 address */ #define LOONGARCH_CSR_IB6MASK 0x3c1 /* inst breakpoint 6 mask */ -#define LOONGARCH_CSR_IB6CTL 0x3c2 /* inst breakpoint 6 control */ +#define LOONGARCH_CSR_IB6CTRL 0x3c2 /* inst breakpoint 6 control */ #define LOONGARCH_CSR_IB6ASID 0x3c3 /* inst breakpoint 6 asid */ #define LOONGARCH_CSR_IB7ADDR 0x3c8 /* inst breakpoint 7 address */ #define LOONGARCH_CSR_IB7MASK 0x3c9 /* inst breakpoint 7 mask */ -#define LOONGARCH_CSR_IB7CTL 0x3ca /* inst breakpoint 7 control */ +#define LOONGARCH_CSR_IB7CTRL 0x3ca /* inst breakpoint 7 control */ #define LOONGARCH_CSR_IB7ASID 0x3cb /* inst breakpoint 7 asid */ #define LOONGARCH_CSR_DEBUG 0x500 /* debug config */ diff --git a/arch/loongarch/include/asm/processor.h b/arch/loongarch/include/asm/processor.h index 7184f1dc61f2..5edf78c3a5b4 100644 --- a/arch/loongarch/include/asm/processor.h +++ b/arch/loongarch/include/asm/processor.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -127,10 +128,14 @@ struct thread_struct { struct loongarch_vdso_info *vdso; /* - * FPU & vector registers, must be at last because - * they are conditionally copied at fork(). + * FPU & vector registers, must be at the last of inherited + * context because they are conditionally copied at fork(). */ struct loongarch_fpu fpu FPU_ALIGN; + + /* Hardware breakpoints pinned to this task. */ + struct perf_event *hbp_break[LOONGARCH_MAX_BRP]; + struct perf_event *hbp_watch[LOONGARCH_MAX_WRP]; }; #define thread_saved_ra(tsk) (tsk->thread.sched_ra) @@ -172,6 +177,8 @@ struct thread_struct { .fcc = 0, \ .fpr = {{{0,},},}, \ }, \ + .hbp_break = {0}, \ + .hbp_watch = {0}, \ } struct task_struct; @@ -184,10 +191,6 @@ extern unsigned long boot_option_idle_override; */ extern void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp); -static inline void flush_thread(void) -{ -} - unsigned long __get_wchan(struct task_struct *p); #define __KSTK_TOS(tsk) ((unsigned long)task_stack_page(tsk) + \ diff --git a/arch/loongarch/include/asm/switch_to.h b/arch/loongarch/include/asm/switch_to.h index 43a5ab162d38..24e3094bebab 100644 --- a/arch/loongarch/include/asm/switch_to.h +++ b/arch/loongarch/include/asm/switch_to.h @@ -34,6 +34,7 @@ extern asmlinkage struct task_struct *__switch_to(struct task_struct *prev, #define switch_to(prev, next, last) \ do { \ lose_fpu_inatomic(1, prev); \ + hw_breakpoint_thread_switch(next); \ (last) = __switch_to(prev, next, task_thread_info(next), \ __builtin_return_address(0), __builtin_frame_address(0)); \ } while (0) diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index 94d33dbd4912..bbb6b047f5db 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -50,5 +50,6 @@ obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o +obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS) diff --git a/arch/loongarch/kernel/hw_breakpoint.c b/arch/loongarch/kernel/hw_breakpoint.c new file mode 100644 index 000000000000..b88efe6c672e --- /dev/null +++ b/arch/loongarch/kernel/hw_breakpoint.c @@ -0,0 +1,523 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022-2023 Loongson Technology Corporation Limited + */ +#define pr_fmt(fmt) "hw-breakpoint: " fmt + +#include +#include +#include + +#include + +/* Breakpoint currently in use for each BRP. */ +static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[LOONGARCH_MAX_BRP]); + +/* Watchpoint currently in use for each WRP. */ +static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[LOONGARCH_MAX_WRP]); + +int hw_breakpoint_slots(int type) +{ + /* + * We can be called early, so don't rely on + * our static variables being initialised. + */ + switch (type) { + case TYPE_INST: + return get_num_brps(); + case TYPE_DATA: + return get_num_wrps(); + default: + pr_warn("unknown slot type: %d\n", type); + return 0; + } +} + +#define READ_WB_REG_CASE(OFF, N, REG, T, VAL) \ + case (OFF + N): \ + LOONGARCH_CSR_WATCH_READ(N, REG, T, VAL); \ + break + +#define WRITE_WB_REG_CASE(OFF, N, REG, T, VAL) \ + case (OFF + N): \ + LOONGARCH_CSR_WATCH_WRITE(N, REG, T, VAL); \ + break + +#define GEN_READ_WB_REG_CASES(OFF, REG, T, VAL) \ + READ_WB_REG_CASE(OFF, 0, REG, T, VAL); \ + READ_WB_REG_CASE(OFF, 1, REG, T, VAL); \ + READ_WB_REG_CASE(OFF, 2, REG, T, VAL); \ + READ_WB_REG_CASE(OFF, 3, REG, T, VAL); \ + READ_WB_REG_CASE(OFF, 4, REG, T, VAL); \ + READ_WB_REG_CASE(OFF, 5, REG, T, VAL); \ + READ_WB_REG_CASE(OFF, 6, REG, T, VAL); \ + READ_WB_REG_CASE(OFF, 7, REG, T, VAL); + +#define GEN_WRITE_WB_REG_CASES(OFF, REG, T, VAL) \ + WRITE_WB_REG_CASE(OFF, 0, REG, T, VAL); \ + WRITE_WB_REG_CASE(OFF, 1, REG, T, VAL); \ + WRITE_WB_REG_CASE(OFF, 2, REG, T, VAL); \ + WRITE_WB_REG_CASE(OFF, 3, REG, T, VAL); \ + WRITE_WB_REG_CASE(OFF, 4, REG, T, VAL); \ + WRITE_WB_REG_CASE(OFF, 5, REG, T, VAL); \ + WRITE_WB_REG_CASE(OFF, 6, REG, T, VAL); \ + WRITE_WB_REG_CASE(OFF, 7, REG, T, VAL); + +static u64 read_wb_reg(int reg, int n, int t) +{ + u64 val = 0; + + switch (reg + n) { + GEN_READ_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val); + GEN_READ_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val); + GEN_READ_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val); + GEN_READ_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val); + default: + pr_warn("Attempt to read from unknown breakpoint register %d\n", n); + } + + return val; +} +NOKPROBE_SYMBOL(read_wb_reg); + +static void write_wb_reg(int reg, int n, int t, u64 val) +{ + switch (reg + n) { + GEN_WRITE_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val); + GEN_WRITE_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val); + GEN_WRITE_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val); + GEN_WRITE_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val); + default: + pr_warn("Attempt to write to unknown breakpoint register %d\n", n); + } +} +NOKPROBE_SYMBOL(write_wb_reg); + +enum hw_breakpoint_ops { + HW_BREAKPOINT_INSTALL, + HW_BREAKPOINT_UNINSTALL, +}; + +/* + * hw_breakpoint_slot_setup - Find and setup a perf slot according to operations + * + * @slots: pointer to array of slots + * @max_slots: max number of slots + * @bp: perf_event to setup + * @ops: operation to be carried out on the slot + * + * Return: + * slot index on success + * -ENOSPC if no slot is available/matches + * -EINVAL on wrong operations parameter + */ + +static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, + struct perf_event *bp, enum hw_breakpoint_ops ops) +{ + int i; + struct perf_event **slot; + + for (i = 0; i < max_slots; ++i) { + slot = &slots[i]; + switch (ops) { + case HW_BREAKPOINT_INSTALL: + if (!*slot) { + *slot = bp; + return i; + } + break; + case HW_BREAKPOINT_UNINSTALL: + if (*slot == bp) { + *slot = NULL; + return i; + } + break; + default: + pr_warn_once("Unhandled hw breakpoint ops %d\n", ops); + return -EINVAL; + } + } + + return -ENOSPC; +} + +void ptrace_hw_copy_thread(struct task_struct *tsk) +{ + memset(tsk->thread.hbp_break, 0, sizeof(tsk->thread.hbp_break)); + memset(tsk->thread.hbp_watch, 0, sizeof(tsk->thread.hbp_watch)); +} + +/* + * Unregister breakpoints from this task and reset the pointers in the thread_struct. + */ +void flush_ptrace_hw_breakpoint(struct task_struct *tsk) +{ +} + +static int hw_breakpoint_control(struct perf_event *bp, + enum hw_breakpoint_ops ops) +{ + u32 ctrl; + int i, max_slots, enable; + struct perf_event **slots; + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + + if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { + /* Breakpoint */ + slots = this_cpu_ptr(bp_on_reg); + max_slots = boot_cpu_data.watch_ireg_count; + } else { + /* Watchpoint */ + slots = this_cpu_ptr(wp_on_reg); + max_slots = boot_cpu_data.watch_dreg_count; + } + + i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops); + + if (WARN_ONCE(i < 0, "Can't find any breakpoint slot")) + return i; + + switch (ops) { + case HW_BREAKPOINT_INSTALL: + /* Set the FWPnCFG/MWPnCFG 1~4 register. */ + write_wb_reg(CSR_CFG_ADDR, i, 0, info->address); + write_wb_reg(CSR_CFG_ADDR, i, 1, info->address); + write_wb_reg(CSR_CFG_MASK, i, 0, info->mask); + write_wb_reg(CSR_CFG_MASK, i, 1, info->mask); + write_wb_reg(CSR_CFG_ASID, i, 0, 0); + write_wb_reg(CSR_CFG_ASID, i, 1, 0); + if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { + write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); + } else { + ctrl = encode_ctrl_reg(info->ctrl); + write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl | CTRL_PLV_ENABLE | + 1 << MWPnCFG3_LoadEn | 1 << MWPnCFG3_StoreEn); + } + enable = csr_read64(LOONGARCH_CSR_CRMD); + csr_write64(CSR_CRMD_WE | enable, LOONGARCH_CSR_CRMD); + break; + case HW_BREAKPOINT_UNINSTALL: + /* Reset the FWPnCFG/MWPnCFG 1~4 register. */ + write_wb_reg(CSR_CFG_ADDR, i, 0, 0); + write_wb_reg(CSR_CFG_ADDR, i, 1, 0); + write_wb_reg(CSR_CFG_MASK, i, 0, 0); + write_wb_reg(CSR_CFG_MASK, i, 1, 0); + write_wb_reg(CSR_CFG_CTRL, i, 0, 0); + write_wb_reg(CSR_CFG_CTRL, i, 1, 0); + write_wb_reg(CSR_CFG_ASID, i, 0, 0); + write_wb_reg(CSR_CFG_ASID, i, 1, 0); + break; + } + + return 0; +} + +/* + * Install a perf counter breakpoint. + */ +int arch_install_hw_breakpoint(struct perf_event *bp) +{ + return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL); +} + +void arch_uninstall_hw_breakpoint(struct perf_event *bp) +{ + hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL); +} + +static int get_hbp_len(u8 hbp_len) +{ + unsigned int len_in_bytes = 0; + + switch (hbp_len) { + case LOONGARCH_BREAKPOINT_LEN_1: + len_in_bytes = 1; + break; + case LOONGARCH_BREAKPOINT_LEN_2: + len_in_bytes = 2; + break; + case LOONGARCH_BREAKPOINT_LEN_4: + len_in_bytes = 4; + break; + case LOONGARCH_BREAKPOINT_LEN_8: + len_in_bytes = 8; + break; + } + + return len_in_bytes; +} + +/* + * Check whether bp virtual address is in kernel space. + */ +int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) +{ + unsigned int len; + unsigned long va; + + va = hw->address; + len = get_hbp_len(hw->ctrl.len); + + return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); +} + +/* + * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. + * Hopefully this will disappear when ptrace can bypass the conversion + * to generic breakpoint descriptions. + */ +int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, + int *gen_len, int *gen_type, int *offset) +{ + /* Type */ + switch (ctrl.type) { + case LOONGARCH_BREAKPOINT_EXECUTE: + *gen_type = HW_BREAKPOINT_X; + break; + case LOONGARCH_BREAKPOINT_LOAD: + *gen_type = HW_BREAKPOINT_R; + break; + case LOONGARCH_BREAKPOINT_STORE: + *gen_type = HW_BREAKPOINT_W; + break; + case LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE: + *gen_type = HW_BREAKPOINT_RW; + break; + default: + return -EINVAL; + } + + if (!ctrl.len) + return -EINVAL; + + *offset = __ffs(ctrl.len); + + /* Len */ + switch (ctrl.len) { + case LOONGARCH_BREAKPOINT_LEN_1: + *gen_len = HW_BREAKPOINT_LEN_1; + break; + case LOONGARCH_BREAKPOINT_LEN_2: + *gen_len = HW_BREAKPOINT_LEN_2; + break; + case LOONGARCH_BREAKPOINT_LEN_4: + *gen_len = HW_BREAKPOINT_LEN_4; + break; + case LOONGARCH_BREAKPOINT_LEN_8: + *gen_len = HW_BREAKPOINT_LEN_8; + break; + default: + return -EINVAL; + } + + return 0; +} + +/* + * Construct an arch_hw_breakpoint from a perf_event. + */ +static int arch_build_bp_info(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) +{ + /* Type */ + switch (attr->bp_type) { + case HW_BREAKPOINT_X: + hw->ctrl.type = LOONGARCH_BREAKPOINT_EXECUTE; + break; + case HW_BREAKPOINT_R: + hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD; + break; + case HW_BREAKPOINT_W: + hw->ctrl.type = LOONGARCH_BREAKPOINT_STORE; + break; + case HW_BREAKPOINT_RW: + hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE; + break; + default: + return -EINVAL; + } + + /* Len */ + switch (attr->bp_len) { + case HW_BREAKPOINT_LEN_1: + hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_1; + break; + case HW_BREAKPOINT_LEN_2: + hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_2; + break; + case HW_BREAKPOINT_LEN_4: + hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_4; + break; + case HW_BREAKPOINT_LEN_8: + hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_8; + break; + default: + return -EINVAL; + } + + /* Address */ + hw->address = attr->bp_addr; + + return 0; +} + +/* + * Validate the arch-specific HW Breakpoint register settings. + */ +int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) +{ + int ret; + u64 alignment_mask, offset; + + /* Build the arch_hw_breakpoint. */ + ret = arch_build_bp_info(bp, attr, hw); + if (ret) + return ret; + + if (hw->ctrl.type != LOONGARCH_BREAKPOINT_EXECUTE) + alignment_mask = 0x7; + offset = hw->address & alignment_mask; + + hw->address &= ~alignment_mask; + hw->ctrl.len <<= offset; + + return 0; +} + +static void update_bp_registers(struct pt_regs *regs, int enable, int type) +{ + u32 ctrl; + int i, max_slots; + struct perf_event **slots; + struct arch_hw_breakpoint *info; + + switch (type) { + case 0: + slots = this_cpu_ptr(bp_on_reg); + max_slots = boot_cpu_data.watch_ireg_count; + break; + case 1: + slots = this_cpu_ptr(wp_on_reg); + max_slots = boot_cpu_data.watch_dreg_count; + break; + default: + return; + } + + for (i = 0; i < max_slots; ++i) { + if (!slots[i]) + continue; + + info = counter_arch_bp(slots[i]); + if (enable) { + if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) { + write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); + write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); + } else { + ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1); + if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD) + ctrl |= 0x1 << MWPnCFG3_LoadEn; + if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE) + ctrl |= 0x1 << MWPnCFG3_StoreEn; + write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl); + } + regs->csr_prmd |= CSR_PRMD_PWE; + } else { + if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) { + write_wb_reg(CSR_CFG_CTRL, i, 0, 0); + } else { + ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1); + if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD) + ctrl &= ~0x1 << MWPnCFG3_LoadEn; + if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE) + ctrl &= ~0x1 << MWPnCFG3_StoreEn; + write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl); + } + regs->csr_prmd &= ~CSR_PRMD_PWE; + } + } +} +NOKPROBE_SYMBOL(update_bp_registers); + +/* + * Debug exception handlers. + */ +void breakpoint_handler(struct pt_regs *regs) +{ + int i; + struct perf_event *bp, **slots; + + slots = this_cpu_ptr(bp_on_reg); + + for (i = 0; i < boot_cpu_data.watch_ireg_count; ++i) { + bp = slots[i]; + if (bp == NULL) + continue; + perf_bp_event(bp, regs); + } + update_bp_registers(regs, 0, 0); +} +NOKPROBE_SYMBOL(breakpoint_handler); + +void watchpoint_handler(struct pt_regs *regs) +{ + int i; + struct perf_event *wp, **slots; + + slots = this_cpu_ptr(wp_on_reg); + + for (i = 0; i < boot_cpu_data.watch_dreg_count; ++i) { + wp = slots[i]; + if (wp == NULL) + continue; + perf_bp_event(wp, regs); + } + update_bp_registers(regs, 0, 1); +} +NOKPROBE_SYMBOL(watchpoint_handler); + +static int __init arch_hw_breakpoint_init(void) +{ + int cpu; + + boot_cpu_data.watch_ireg_count = get_num_brps(); + boot_cpu_data.watch_dreg_count = get_num_wrps(); + + pr_info("Found %d breakpoint and %d watchpoint registers.\n", + boot_cpu_data.watch_ireg_count, boot_cpu_data.watch_dreg_count); + + for (cpu = 1; cpu < NR_CPUS; cpu++) { + cpu_data[cpu].watch_ireg_count = boot_cpu_data.watch_ireg_count; + cpu_data[cpu].watch_dreg_count = boot_cpu_data.watch_dreg_count; + } + + return 0; +} +arch_initcall(arch_hw_breakpoint_init); + +void hw_breakpoint_thread_switch(struct task_struct *next) +{ + struct pt_regs *regs = task_pt_regs(next); + + /* Update breakpoints */ + update_bp_registers(regs, 1, 0); + /* Update watchpoints */ + update_bp_registers(regs, 1, 1); +} + +void hw_breakpoint_pmu_read(struct perf_event *bp) +{ +} + +/* + * Dummy function to register with die_notifier. + */ +int hw_breakpoint_exceptions_notify(struct notifier_block *unused, + unsigned long val, void *data) +{ + return NOTIFY_DONE; +} diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c index edfd220a3737..fa2443c7afb2 100644 --- a/arch/loongarch/kernel/process.c +++ b/arch/loongarch/kernel/process.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -96,6 +97,11 @@ void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) regs->regs[3] = sp; } +void flush_thread(void) +{ + flush_ptrace_hw_breakpoint(current); +} + void exit_thread(struct task_struct *tsk) { } @@ -181,6 +187,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) childregs->regs[2] = tls; out: + ptrace_hw_copy_thread(p); clear_tsk_thread_flag(p, TIF_USEDFPU); clear_tsk_thread_flag(p, TIF_USEDSIMD); clear_tsk_thread_flag(p, TIF_LSX_CTX_LIVE); diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c index 05511203732c..ba67d787f0d7 100644 --- a/arch/loongarch/kernel/traps.c +++ b/arch/loongarch/kernel/traps.c @@ -511,7 +511,17 @@ out_sigsegv: asmlinkage void noinstr do_watch(struct pt_regs *regs) { + irqentry_state_t state = irqentry_enter(regs); + +#ifdef CONFIG_HAVE_HW_BREAKPOINT + breakpoint_handler(regs); + watchpoint_handler(regs); + force_sig(SIGTRAP); +#else pr_warn("Hardware watch point handler not implemented!\n"); +#endif + + irqentry_exit(regs, state); } asmlinkage void noinstr do_ri(struct pt_regs *regs) -- cgit v1.2.3 From 1a69f7a161a78aead07cd4b811d796950e892fa4 Mon Sep 17 00:00:00 2001 From: Qing Zhang Date: Sat, 25 Feb 2023 15:52:57 +0800 Subject: LoongArch: ptrace: Expose hardware breakpoints to debuggers Implement the regset-based ptrace interface that exposes hardware breakpoints to user-space debuggers to query and set instruction and data breakpoints. Signed-off-by: Qing Zhang Signed-off-by: Huacai Chen --- arch/loongarch/include/uapi/asm/ptrace.h | 9 + arch/loongarch/kernel/ptrace.c | 402 +++++++++++++++++++++++++++++++ 2 files changed, 411 insertions(+) (limited to 'arch/loongarch') diff --git a/arch/loongarch/include/uapi/asm/ptrace.h b/arch/loongarch/include/uapi/asm/ptrace.h index 083193f4a5d5..cc48ed262021 100644 --- a/arch/loongarch/include/uapi/asm/ptrace.h +++ b/arch/loongarch/include/uapi/asm/ptrace.h @@ -46,6 +46,15 @@ struct user_fp_state { uint32_t fcsr; }; +struct user_watch_state { + uint16_t dbg_info; + struct { + uint64_t addr; + uint64_t mask; + uint32_t ctrl; + } dbg_regs[8]; +}; + #define PTRACE_SYSEMU 0x1f #define PTRACE_SYSEMU_SINGLESTEP 0x20 diff --git a/arch/loongarch/kernel/ptrace.c b/arch/loongarch/kernel/ptrace.c index dc2b82ea894c..11b491395636 100644 --- a/arch/loongarch/kernel/ptrace.c +++ b/arch/loongarch/kernel/ptrace.c @@ -20,7 +20,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -246,6 +248,384 @@ static int cfg_set(struct task_struct *target, return 0; } +#ifdef CONFIG_HAVE_HW_BREAKPOINT + +/* + * Handle hitting a HW-breakpoint. + */ +static void ptrace_hbptriggered(struct perf_event *bp, + struct perf_sample_data *data, + struct pt_regs *regs) +{ + int i; + struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp); + + for (i = 0; i < LOONGARCH_MAX_BRP; ++i) + if (current->thread.hbp_break[i] == bp) + break; + + for (i = 0; i < LOONGARCH_MAX_WRP; ++i) + if (current->thread.hbp_watch[i] == bp) + break; + + force_sig_ptrace_errno_trap(i, (void __user *)bkpt->address); +} + +static struct perf_event *ptrace_hbp_get_event(unsigned int note_type, + struct task_struct *tsk, + unsigned long idx) +{ + struct perf_event *bp; + + switch (note_type) { + case NT_LOONGARCH_HW_BREAK: + if (idx >= LOONGARCH_MAX_BRP) + return ERR_PTR(-EINVAL); + idx = array_index_nospec(idx, LOONGARCH_MAX_BRP); + bp = tsk->thread.hbp_break[idx]; + break; + case NT_LOONGARCH_HW_WATCH: + if (idx >= LOONGARCH_MAX_WRP) + return ERR_PTR(-EINVAL); + idx = array_index_nospec(idx, LOONGARCH_MAX_WRP); + bp = tsk->thread.hbp_watch[idx]; + break; + } + + return bp; +} + +static int ptrace_hbp_set_event(unsigned int note_type, + struct task_struct *tsk, + unsigned long idx, + struct perf_event *bp) +{ + switch (note_type) { + case NT_LOONGARCH_HW_BREAK: + if (idx >= LOONGARCH_MAX_BRP) + return -EINVAL; + idx = array_index_nospec(idx, LOONGARCH_MAX_BRP); + tsk->thread.hbp_break[idx] = bp; + break; + case NT_LOONGARCH_HW_WATCH: + if (idx >= LOONGARCH_MAX_WRP) + return -EINVAL; + idx = array_index_nospec(idx, LOONGARCH_MAX_WRP); + tsk->thread.hbp_watch[idx] = bp; + break; + } + + return 0; +} + +static struct perf_event *ptrace_hbp_create(unsigned int note_type, + struct task_struct *tsk, + unsigned long idx) +{ + int err, type; + struct perf_event *bp; + struct perf_event_attr attr; + + switch (note_type) { + case NT_LOONGARCH_HW_BREAK: + type = HW_BREAKPOINT_X; + break; + case NT_LOONGARCH_HW_WATCH: + type = HW_BREAKPOINT_RW; + break; + default: + return ERR_PTR(-EINVAL); + } + + ptrace_breakpoint_init(&attr); + + /* + * Initialise fields to sane defaults + * (i.e. values that will pass validation). + */ + attr.bp_addr = 0; + attr.bp_len = HW_BREAKPOINT_LEN_4; + attr.bp_type = type; + attr.disabled = 1; + + bp = register_user_hw_breakpoint(&attr, ptrace_hbptriggered, NULL, tsk); + if (IS_ERR(bp)) + return bp; + + err = ptrace_hbp_set_event(note_type, tsk, idx, bp); + if (err) + return ERR_PTR(err); + + return bp; +} + +static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type, + struct arch_hw_breakpoint_ctrl ctrl, + struct perf_event_attr *attr) +{ + int err, len, type, offset; + + err = arch_bp_generic_fields(ctrl, &len, &type, &offset); + if (err) + return err; + + switch (note_type) { + case NT_LOONGARCH_HW_BREAK: + if ((type & HW_BREAKPOINT_X) != type) + return -EINVAL; + break; + case NT_LOONGARCH_HW_WATCH: + if ((type & HW_BREAKPOINT_RW) != type) + return -EINVAL; + break; + default: + return -EINVAL; + } + + attr->bp_len = len; + attr->bp_type = type; + attr->bp_addr += offset; + + return 0; +} + +static int ptrace_hbp_get_resource_info(unsigned int note_type, u16 *info) +{ + u8 num; + u16 reg = 0; + + switch (note_type) { + case NT_LOONGARCH_HW_BREAK: + num = hw_breakpoint_slots(TYPE_INST); + break; + case NT_LOONGARCH_HW_WATCH: + num = hw_breakpoint_slots(TYPE_DATA); + break; + default: + return -EINVAL; + } + + *info = reg | num; + + return 0; +} + +static struct perf_event *ptrace_hbp_get_initialised_bp(unsigned int note_type, + struct task_struct *tsk, + unsigned long idx) +{ + struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx); + + if (!bp) + bp = ptrace_hbp_create(note_type, tsk, idx); + + return bp; +} + +static int ptrace_hbp_get_ctrl(unsigned int note_type, + struct task_struct *tsk, + unsigned long idx, u32 *ctrl) +{ + struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx); + + if (IS_ERR(bp)) + return PTR_ERR(bp); + + *ctrl = bp ? encode_ctrl_reg(counter_arch_bp(bp)->ctrl) : 0; + + return 0; +} + +static int ptrace_hbp_get_mask(unsigned int note_type, + struct task_struct *tsk, + unsigned long idx, u64 *mask) +{ + struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx); + + if (IS_ERR(bp)) + return PTR_ERR(bp); + + *mask = bp ? counter_arch_bp(bp)->mask : 0; + + return 0; +} + +static int ptrace_hbp_get_addr(unsigned int note_type, + struct task_struct *tsk, + unsigned long idx, u64 *addr) +{ + struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx); + + if (IS_ERR(bp)) + return PTR_ERR(bp); + + *addr = bp ? counter_arch_bp(bp)->address : 0; + + return 0; +} + +static int ptrace_hbp_set_ctrl(unsigned int note_type, + struct task_struct *tsk, + unsigned long idx, u32 uctrl) +{ + int err; + struct perf_event *bp; + struct perf_event_attr attr; + struct arch_hw_breakpoint_ctrl ctrl; + + bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx); + if (IS_ERR(bp)) + return PTR_ERR(bp); + + attr = bp->attr; + decode_ctrl_reg(uctrl, &ctrl); + err = ptrace_hbp_fill_attr_ctrl(note_type, ctrl, &attr); + if (err) + return err; + + return modify_user_hw_breakpoint(bp, &attr); +} + +static int ptrace_hbp_set_mask(unsigned int note_type, + struct task_struct *tsk, + unsigned long idx, u64 mask) +{ + struct perf_event *bp; + struct perf_event_attr attr; + struct arch_hw_breakpoint *info; + + bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx); + if (IS_ERR(bp)) + return PTR_ERR(bp); + + attr = bp->attr; + info = counter_arch_bp(bp); + info->mask = mask; + + return modify_user_hw_breakpoint(bp, &attr); +} + +static int ptrace_hbp_set_addr(unsigned int note_type, + struct task_struct *tsk, + unsigned long idx, u64 addr) +{ + struct perf_event *bp; + struct perf_event_attr attr; + + bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx); + if (IS_ERR(bp)) + return PTR_ERR(bp); + + attr = bp->attr; + attr.bp_addr = addr; + + return modify_user_hw_breakpoint(bp, &attr); +} + +#define PTRACE_HBP_CTRL_SZ sizeof(u32) +#define PTRACE_HBP_ADDR_SZ sizeof(u64) +#define PTRACE_HBP_MASK_SZ sizeof(u64) + +static int hw_break_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + u16 info; + u32 ctrl; + u64 addr, mask; + int ret, idx = 0; + unsigned int note_type = regset->core_note_type; + + /* Resource info */ + ret = ptrace_hbp_get_resource_info(note_type, &info); + if (ret) + return ret; + + membuf_write(&to, &info, sizeof(info)); + + /* (address, ctrl) registers */ + while (to.left) { + ret = ptrace_hbp_get_addr(note_type, target, idx, &addr); + if (ret) + return ret; + + ret = ptrace_hbp_get_mask(note_type, target, idx, &mask); + if (ret) + return ret; + + ret = ptrace_hbp_get_ctrl(note_type, target, idx, &ctrl); + if (ret) + return ret; + + membuf_store(&to, addr); + membuf_store(&to, mask); + membuf_store(&to, ctrl); + idx++; + } + + return 0; +} + +static int hw_break_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + u32 ctrl; + u64 addr, mask; + int ret, idx = 0, offset, limit; + unsigned int note_type = regset->core_note_type; + + /* Resource info */ + offset = offsetof(struct user_watch_state, dbg_regs); + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 0, offset); + + /* (address, ctrl) registers */ + limit = regset->n * regset->size; + while (count && offset < limit) { + if (count < PTRACE_HBP_ADDR_SZ) + return -EINVAL; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &addr, + offset, offset + PTRACE_HBP_ADDR_SZ); + if (ret) + return ret; + + ret = ptrace_hbp_set_addr(note_type, target, idx, addr); + if (ret) + return ret; + offset += PTRACE_HBP_ADDR_SZ; + + if (!count) + break; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &mask, + offset, offset + PTRACE_HBP_ADDR_SZ); + if (ret) + return ret; + + ret = ptrace_hbp_set_mask(note_type, target, idx, mask); + if (ret) + return ret; + offset += PTRACE_HBP_MASK_SZ; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &mask, + offset, offset + PTRACE_HBP_MASK_SZ); + if (ret) + return ret; + + ret = ptrace_hbp_set_ctrl(note_type, target, idx, ctrl); + if (ret) + return ret; + offset += PTRACE_HBP_CTRL_SZ; + idx++; + } + + return 0; +} + +#endif + struct pt_regs_offset { const char *name; int offset; @@ -319,6 +699,10 @@ enum loongarch_regset { REGSET_GPR, REGSET_FPR, REGSET_CPUCFG, +#ifdef CONFIG_HAVE_HW_BREAKPOINT + REGSET_HW_BREAK, + REGSET_HW_WATCH, +#endif }; static const struct user_regset loongarch64_regsets[] = { @@ -346,6 +730,24 @@ static const struct user_regset loongarch64_regsets[] = { .regset_get = cfg_get, .set = cfg_set, }, +#ifdef CONFIG_HAVE_HW_BREAKPOINT + [REGSET_HW_BREAK] = { + .core_note_type = NT_LOONGARCH_HW_BREAK, + .n = sizeof(struct user_watch_state) / sizeof(u32), + .size = sizeof(u32), + .align = sizeof(u32), + .regset_get = hw_break_get, + .set = hw_break_set, + }, + [REGSET_HW_WATCH] = { + .core_note_type = NT_LOONGARCH_HW_WATCH, + .n = sizeof(struct user_watch_state) / sizeof(u32), + .size = sizeof(u32), + .align = sizeof(u32), + .regset_get = hw_break_get, + .set = hw_break_set, + }, +#endif }; static const struct user_regset_view user_loongarch64_view = { -- cgit v1.2.3 From 356bd6f23682f11f7afe923d86c7f5f852b97fb2 Mon Sep 17 00:00:00 2001 From: Qing Zhang Date: Sat, 25 Feb 2023 15:52:57 +0800 Subject: LoongArch: ptrace: Add function argument access API Add regs_get_argument() which returns N th argument of the function call, This enables ftrace kprobe events to access kernel function arguments via $argN syntax for later use. E.g.: echo 'p bio_add_page arg1=$arg1' > kprobe_events bash: echo: write error: Invalid argument Signed-off-by: Qing Zhang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/ptrace.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) (limited to 'arch/loongarch') diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 97423f86384b..277ce9b9414e 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -98,6 +98,7 @@ config LOONGARCH select HAVE_EXIT_THREAD select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER select HAVE_GENERIC_VDSO diff --git a/arch/loongarch/include/asm/ptrace.h b/arch/loongarch/include/asm/ptrace.h index 59c4608de91d..5d3872131866 100644 --- a/arch/loongarch/include/asm/ptrace.h +++ b/arch/loongarch/include/asm/ptrace.h @@ -109,6 +109,40 @@ static inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsi struct task_struct; +/** + * regs_get_kernel_argument() - get Nth function argument in kernel + * @regs: pt_regs of that context + * @n: function argument number (start from 0) + * + * regs_get_argument() returns @n th argument of the function call. + * Note that this chooses most probably assignment, in some case + * it can be incorrect. + * This is expected to be called from kprobes or ftrace with regs + * where the top of stack is the return address. + */ +static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs, + unsigned int n) +{ +#define NR_REG_ARGUMENTS 8 + static const unsigned int args[] = { + offsetof(struct pt_regs, regs[4]), + offsetof(struct pt_regs, regs[5]), + offsetof(struct pt_regs, regs[6]), + offsetof(struct pt_regs, regs[7]), + offsetof(struct pt_regs, regs[8]), + offsetof(struct pt_regs, regs[9]), + offsetof(struct pt_regs, regs[10]), + offsetof(struct pt_regs, regs[11]), + }; + + if (n < NR_REG_ARGUMENTS) + return regs_get_register(regs, args[n]); + else { + n -= NR_REG_ARGUMENTS; + return regs_get_kernel_stack_nth(regs, n); + } +} + /* * Does the process account for user or for system time? */ -- cgit v1.2.3 From 424421a7f34c1222d20a6c279f13b9caa71ecc83 Mon Sep 17 00:00:00 2001 From: Qing Zhang Date: Sat, 25 Feb 2023 15:52:57 +0800 Subject: LoongArch: ptrace: Add hardware single step support Use the generic ptrace_resume code for PTRACE_SYSCALL, PTRACE_CONT, PTRACE_KILL and PTRACE_SINGLESTEP handling. This implies defining arch_has_single_step() and implementing the user_enable_single_step() and user_disable_single_step() functions. LoongArch cannot do hardware single-stepping per se, the hardware single-stepping it is achieved by configuring the instruction fetch watchpoints (FWPS) and specifies that the next instruction must trigger the watch exception by setting the mask bit. In some scenarios CSR.FWPS.Skip is used to ignore the next hit result, avoid endless repeated triggering of the same watchpoint without canceling it. Signed-off-by: Qing Zhang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/inst.h | 38 ++++++++++++++++++ arch/loongarch/include/asm/loongarch.h | 3 ++ arch/loongarch/include/asm/processor.h | 1 + arch/loongarch/include/asm/ptrace.h | 4 ++ arch/loongarch/kernel/hw_breakpoint.c | 33 ++++++++++++++-- arch/loongarch/kernel/ptrace.c | 70 ++++++++++++++++++++++++++++++++++ arch/loongarch/kernel/traps.c | 47 ++++++++++++++++++++--- 7 files changed, 186 insertions(+), 10 deletions(-) (limited to 'arch/loongarch') diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 7eedd83fd0d7..36cc5eb4f852 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -351,6 +351,44 @@ static inline bool is_stack_alloc_ins(union loongarch_instruction *ip) is_imm12_negative(ip->reg2i12_format.immediate); } +static inline bool is_self_loop_ins(union loongarch_instruction *ip, struct pt_regs *regs) +{ + switch (ip->reg0i26_format.opcode) { + case b_op: + case bl_op: + if (ip->reg0i26_format.immediate_l == 0 + && ip->reg0i26_format.immediate_h == 0) + return true; + } + + switch (ip->reg1i21_format.opcode) { + case beqz_op: + case bnez_op: + case bceqz_op: + if (ip->reg1i21_format.immediate_l == 0 + && ip->reg1i21_format.immediate_h == 0) + return true; + } + + switch (ip->reg2i16_format.opcode) { + case beq_op: + case bne_op: + case blt_op: + case bge_op: + case bltu_op: + case bgeu_op: + if (ip->reg2i16_format.immediate == 0) + return true; + break; + case jirl_op: + if (regs->regs[ip->reg2i16_format.rj] + + ((unsigned long)ip->reg2i16_format.immediate << 2) == (unsigned long)ip) + return true; + } + + return false; +} + int larch_insn_read(void *addr, u32 *insnp); int larch_insn_write(void *addr, u32 insn); int larch_insn_patch_text(void *addr, u32 insn); diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index e9aed583a064..65b7dcdea16d 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -1055,6 +1055,9 @@ static __always_inline void iocsr_write64(u64 val, u32 reg) #define LOONGARCH_CSR_DERA 0x501 /* debug era */ #define LOONGARCH_CSR_DESAVE 0x502 /* debug save */ +#define CSR_FWPC_SKIP_SHIFT 16 +#define CSR_FWPC_SKIP (_ULCAST_(1) << CSR_FWPC_SKIP_SHIFT) + /* * CSR_ECFG IM */ diff --git a/arch/loongarch/include/asm/processor.h b/arch/loongarch/include/asm/processor.h index 5edf78c3a5b4..636e1c66398c 100644 --- a/arch/loongarch/include/asm/processor.h +++ b/arch/loongarch/include/asm/processor.h @@ -125,6 +125,7 @@ struct thread_struct { /* Other stuff associated with the thread. */ unsigned long trap_nr; unsigned long error_code; + unsigned long single_step; /* Used by PTRACE_SINGLESTEP */ struct loongarch_vdso_info *vdso; /* diff --git a/arch/loongarch/include/asm/ptrace.h b/arch/loongarch/include/asm/ptrace.h index 5d3872131866..6eab4393b108 100644 --- a/arch/loongarch/include/asm/ptrace.h +++ b/arch/loongarch/include/asm/ptrace.h @@ -183,4 +183,8 @@ static inline void user_stack_pointer_set(struct pt_regs *regs, regs->regs[3] = val; } +#ifdef CONFIG_HAVE_HW_BREAKPOINT +#define arch_has_single_step() (1) +#endif + #endif /* _ASM_PTRACE_H */ diff --git a/arch/loongarch/kernel/hw_breakpoint.c b/arch/loongarch/kernel/hw_breakpoint.c index b88efe6c672e..2406c95b34cc 100644 --- a/arch/loongarch/kernel/hw_breakpoint.c +++ b/arch/loongarch/kernel/hw_breakpoint.c @@ -153,6 +153,22 @@ void ptrace_hw_copy_thread(struct task_struct *tsk) */ void flush_ptrace_hw_breakpoint(struct task_struct *tsk) { + int i; + struct thread_struct *t = &tsk->thread; + + for (i = 0; i < LOONGARCH_MAX_BRP; i++) { + if (t->hbp_break[i]) { + unregister_hw_breakpoint(t->hbp_break[i]); + t->hbp_break[i] = NULL; + } + } + + for (i = 0; i < LOONGARCH_MAX_WRP; i++) { + if (t->hbp_watch[i]) { + unregister_hw_breakpoint(t->hbp_watch[i]); + t->hbp_watch[i] = NULL; + } + } } static int hw_breakpoint_control(struct perf_event *bp, @@ -501,12 +517,21 @@ arch_initcall(arch_hw_breakpoint_init); void hw_breakpoint_thread_switch(struct task_struct *next) { + u64 addr, mask; struct pt_regs *regs = task_pt_regs(next); - /* Update breakpoints */ - update_bp_registers(regs, 1, 0); - /* Update watchpoints */ - update_bp_registers(regs, 1, 1); + if (test_tsk_thread_flag(next, TIF_SINGLESTEP)) { + addr = read_wb_reg(CSR_CFG_ADDR, 0, 0); + mask = read_wb_reg(CSR_CFG_MASK, 0, 0); + if (!((regs->csr_era ^ addr) & ~mask)) + csr_write32(CSR_FWPC_SKIP, LOONGARCH_CSR_FWPS); + regs->csr_prmd |= CSR_PRMD_PWE; + } else { + /* Update breakpoints */ + update_bp_registers(regs, 1, 0); + /* Update watchpoints */ + update_bp_registers(regs, 1, 1); + } } void hw_breakpoint_pmu_read(struct perf_event *bp) diff --git a/arch/loongarch/kernel/ptrace.c b/arch/loongarch/kernel/ptrace.c index 11b491395636..06bceae7d104 100644 --- a/arch/loongarch/kernel/ptrace.c +++ b/arch/loongarch/kernel/ptrace.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -41,6 +42,7 @@ #include #include #include +#include #include #include @@ -833,3 +835,71 @@ long arch_ptrace(struct task_struct *child, long request, return ret; } + +#ifdef CONFIG_HAVE_HW_BREAKPOINT +static void ptrace_triggered(struct perf_event *bp, + struct perf_sample_data *data, struct pt_regs *regs) +{ + struct perf_event_attr attr; + + attr = bp->attr; + attr.disabled = true; + modify_user_hw_breakpoint(bp, &attr); +} + +static int set_single_step(struct task_struct *tsk, unsigned long addr) +{ + struct perf_event *bp; + struct perf_event_attr attr; + struct arch_hw_breakpoint *info; + struct thread_struct *thread = &tsk->thread; + + bp = thread->hbp_break[0]; + if (!bp) { + ptrace_breakpoint_init(&attr); + + attr.bp_addr = addr; + attr.bp_len = HW_BREAKPOINT_LEN_8; + attr.bp_type = HW_BREAKPOINT_X; + + bp = register_user_hw_breakpoint(&attr, ptrace_triggered, + NULL, tsk); + if (IS_ERR(bp)) + return PTR_ERR(bp); + + thread->hbp_break[0] = bp; + } else { + int err; + + attr = bp->attr; + attr.bp_addr = addr; + + /* Reenable breakpoint */ + attr.disabled = false; + err = modify_user_hw_breakpoint(bp, &attr); + if (unlikely(err)) + return err; + + csr_write64(attr.bp_addr, LOONGARCH_CSR_IB0ADDR); + } + info = counter_arch_bp(bp); + info->mask = TASK_SIZE - 1; + + return 0; +} + +/* ptrace API */ +void user_enable_single_step(struct task_struct *task) +{ + struct thread_info *ti = task_thread_info(task); + + set_single_step(task, task_pt_regs(task)->csr_era); + task->thread.single_step = task_pt_regs(task)->csr_era; + set_ti_thread_flag(ti, TIF_SINGLESTEP); +} + +void user_disable_single_step(struct task_struct *task) +{ + clear_tsk_thread_flag(task, TIF_SINGLESTEP); +} +#endif diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c index ba67d787f0d7..1d47747e2154 100644 --- a/arch/loongarch/kernel/traps.c +++ b/arch/loongarch/kernel/traps.c @@ -513,14 +513,49 @@ asmlinkage void noinstr do_watch(struct pt_regs *regs) { irqentry_state_t state = irqentry_enter(regs); -#ifdef CONFIG_HAVE_HW_BREAKPOINT - breakpoint_handler(regs); - watchpoint_handler(regs); - force_sig(SIGTRAP); -#else +#ifndef CONFIG_HAVE_HW_BREAKPOINT pr_warn("Hardware watch point handler not implemented!\n"); -#endif +#else + if (test_tsk_thread_flag(current, TIF_SINGLESTEP)) { + int llbit = (csr_read32(LOONGARCH_CSR_LLBCTL) & 0x1); + unsigned long pc = instruction_pointer(regs); + union loongarch_instruction *ip = (union loongarch_instruction *)pc; + + if (llbit) { + /* + * When the ll-sc combo is encountered, it is regarded as an single + * instruction. So don't clear llbit and reset CSR.FWPS.Skip until + * the llsc execution is completed. + */ + csr_write32(CSR_FWPC_SKIP, LOONGARCH_CSR_FWPS); + csr_write32(CSR_LLBCTL_KLO, LOONGARCH_CSR_LLBCTL); + goto out; + } + if (pc == current->thread.single_step) { + /* + * Certain insns are occasionally not skipped when CSR.FWPS.Skip is + * set, such as fld.d/fst.d. So singlestep needs to compare whether + * the csr_era is equal to the value of singlestep which last time set. + */ + if (!is_self_loop_ins(ip, regs)) { + /* + * Check if the given instruction the target pc is equal to the + * current pc, If yes, then we should not set the CSR.FWPS.SKIP + * bit to break the original instruction stream. + */ + csr_write32(CSR_FWPC_SKIP, LOONGARCH_CSR_FWPS); + goto out; + } + } + } else { + breakpoint_handler(regs); + watchpoint_handler(regs); + } + + force_sig(SIGTRAP); +out: +#endif irqentry_exit(regs, state); } -- cgit v1.2.3 From 9b3441a6b0e1ed4796e7f5a8586c4d6023c4e3ab Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Sat, 25 Feb 2023 15:52:57 +0800 Subject: LoongArch: Simulate branch and PC* instructions According to LoongArch Reference Manual, simulate branch and PC* instructions, this is preparation for later patch. Link: https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#branch-instructions Link: https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#_pcaddi_pcaddu121_pcaddu18l_pcalau12i Tested-by: Jeff Xie Co-developed-by: Jinyang He Signed-off-by: Jinyang He Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/inst.h | 5 ++ arch/loongarch/include/asm/ptrace.h | 1 + arch/loongarch/kernel/inst.c | 123 ++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+) (limited to 'arch/loongarch') diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 36cc5eb4f852..3ade9ff2bb8f 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -7,6 +7,7 @@ #include #include +#include #define INSN_NOP 0x03400000 #define INSN_BREAK 0x002a0000 @@ -32,6 +33,7 @@ enum reg1i20_op { lu12iw_op = 0x0a, lu32id_op = 0x0b, pcaddi_op = 0x0c, + pcalau12i_op = 0x0d, pcaddu12i_op = 0x0e, pcaddu18i_op = 0x0f, }; @@ -389,6 +391,9 @@ static inline bool is_self_loop_ins(union loongarch_instruction *ip, struct pt_r return false; } +void simu_pc(struct pt_regs *regs, union loongarch_instruction insn); +void simu_branch(struct pt_regs *regs, union loongarch_instruction insn); + int larch_insn_read(void *addr, u32 *insnp); int larch_insn_write(void *addr, u32 insn); int larch_insn_patch_text(void *addr, u32 insn); diff --git a/arch/loongarch/include/asm/ptrace.h b/arch/loongarch/include/asm/ptrace.h index 6eab4393b108..d761db943335 100644 --- a/arch/loongarch/include/asm/ptrace.h +++ b/arch/loongarch/include/asm/ptrace.h @@ -6,6 +6,7 @@ #define _ASM_PTRACE_H #include +#include #include #include diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c index badc59087042..258ef267cd30 100644 --- a/arch/loongarch/kernel/inst.c +++ b/arch/loongarch/kernel/inst.c @@ -10,6 +10,129 @@ static DEFINE_RAW_SPINLOCK(patch_lock); +void simu_pc(struct pt_regs *regs, union loongarch_instruction insn) +{ + unsigned long pc = regs->csr_era; + unsigned int rd = insn.reg1i20_format.rd; + unsigned int imm = insn.reg1i20_format.immediate; + + if (pc & 3) { + pr_warn("%s: invalid pc 0x%lx\n", __func__, pc); + return; + } + + switch (insn.reg1i20_format.opcode) { + case pcaddi_op: + regs->regs[rd] = pc + sign_extend64(imm << 2, 21); + break; + case pcaddu12i_op: + regs->regs[rd] = pc + sign_extend64(imm << 12, 31); + break; + case pcaddu18i_op: + regs->regs[rd] = pc + sign_extend64(imm << 18, 37); + break; + case pcalau12i_op: + regs->regs[rd] = pc + sign_extend64(imm << 12, 31); + regs->regs[rd] &= ~((1 << 12) - 1); + break; + default: + pr_info("%s: unknown opcode\n", __func__); + return; + } + + regs->csr_era += LOONGARCH_INSN_SIZE; +} + +void simu_branch(struct pt_regs *regs, union loongarch_instruction insn) +{ + unsigned int imm, imm_l, imm_h, rd, rj; + unsigned long pc = regs->csr_era; + + if (pc & 3) { + pr_warn("%s: invalid pc 0x%lx\n", __func__, pc); + return; + } + + imm_l = insn.reg0i26_format.immediate_l; + imm_h = insn.reg0i26_format.immediate_h; + switch (insn.reg0i26_format.opcode) { + case b_op: + regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 27); + return; + case bl_op: + regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 27); + regs->regs[1] = pc + LOONGARCH_INSN_SIZE; + return; + } + + imm_l = insn.reg1i21_format.immediate_l; + imm_h = insn.reg1i21_format.immediate_h; + rj = insn.reg1i21_format.rj; + switch (insn.reg1i21_format.opcode) { + case beqz_op: + if (regs->regs[rj] == 0) + regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 22); + else + regs->csr_era = pc + LOONGARCH_INSN_SIZE; + return; + case bnez_op: + if (regs->regs[rj] != 0) + regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 22); + else + regs->csr_era = pc + LOONGARCH_INSN_SIZE; + return; + } + + imm = insn.reg2i16_format.immediate; + rj = insn.reg2i16_format.rj; + rd = insn.reg2i16_format.rd; + switch (insn.reg2i16_format.opcode) { + case beq_op: + if (regs->regs[rj] == regs->regs[rd]) + regs->csr_era = pc + sign_extend64(imm << 2, 17); + else + regs->csr_era = pc + LOONGARCH_INSN_SIZE; + break; + case bne_op: + if (regs->regs[rj] != regs->regs[rd]) + regs->csr_era = pc + sign_extend64(imm << 2, 17); + else + regs->csr_era = pc + LOONGARCH_INSN_SIZE; + break; + case blt_op: + if ((long)regs->regs[rj] < (long)regs->regs[rd]) + regs->csr_era = pc + sign_extend64(imm << 2, 17); + else + regs->csr_era = pc + LOONGARCH_INSN_SIZE; + break; + case bge_op: + if ((long)regs->regs[rj] >= (long)regs->regs[rd]) + regs->csr_era = pc + sign_extend64(imm << 2, 17); + else + regs->csr_era = pc + LOONGARCH_INSN_SIZE; + break; + case bltu_op: + if (regs->regs[rj] < regs->regs[rd]) + regs->csr_era = pc + sign_extend64(imm << 2, 17); + else + regs->csr_era = pc + LOONGARCH_INSN_SIZE; + break; + case bgeu_op: + if (regs->regs[rj] >= regs->regs[rd]) + regs->csr_era = pc + sign_extend64(imm << 2, 17); + else + regs->csr_era = pc + LOONGARCH_INSN_SIZE; + break; + case jirl_op: + regs->csr_era = regs->regs[rj] + sign_extend64(imm << 2, 17); + regs->regs[rd] = pc + LOONGARCH_INSN_SIZE; + break; + default: + pr_info("%s: unknown opcode\n", __func__); + return; + } +} + int larch_insn_read(void *addr, u32 *insnp) { int ret; -- cgit v1.2.3 From 6d4cc40fb5f58147defc6c0e9d86e6232fe31616 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Sat, 25 Feb 2023 15:52:57 +0800 Subject: LoongArch: Add kprobes support Kprobes allows you to trap at almost any kernel address and execute a callback function, this commit adds kprobes support for LoongArch. Tested-by: Jeff Xie Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/inst.h | 15 ++ arch/loongarch/include/asm/kprobes.h | 58 ++++++ arch/loongarch/kernel/Makefile | 2 + arch/loongarch/kernel/kprobes.c | 379 +++++++++++++++++++++++++++++++++++ arch/loongarch/kernel/traps.c | 14 +- arch/loongarch/mm/fault.c | 3 + 7 files changed, 466 insertions(+), 6 deletions(-) create mode 100644 arch/loongarch/include/asm/kprobes.h create mode 100644 arch/loongarch/kernel/kprobes.c (limited to 'arch/loongarch') diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 277ce9b9414e..bb538b68116d 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -106,6 +106,7 @@ config LOONGARCH select HAVE_IOREMAP_PROT select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_IRQ_TIME_ACCOUNTING + select HAVE_KPROBES select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI select HAVE_PCI diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 3ade9ff2bb8f..a04fe755d719 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -24,6 +24,10 @@ #define ADDR_IMM(addr, INSN) ((addr & ADDR_IMMMASK_##INSN) >> ADDR_IMMSHIFT_##INSN) +enum reg0i15_op { + break_op = 0x54, +}; + enum reg0i26_op { b_op = 0x14, bl_op = 0x15, @@ -180,6 +184,11 @@ enum reg3sa2_op { alsld_op = 0x16, }; +struct reg0i15_format { + unsigned int immediate : 15; + unsigned int opcode : 17; +}; + struct reg0i26_format { unsigned int immediate_h : 10; unsigned int immediate_l : 16; @@ -265,6 +274,7 @@ struct reg3sa2_format { union loongarch_instruction { unsigned int word; + struct reg0i15_format reg0i15_format; struct reg0i26_format reg0i26_format; struct reg1i20_format reg1i20_format; struct reg1i21_format reg1i21_format; @@ -323,6 +333,11 @@ static inline bool is_imm_negative(unsigned long val, unsigned int bit) return val & (1UL << (bit - 1)); } +static inline bool is_break_ins(union loongarch_instruction *ip) +{ + return ip->reg0i15_format.opcode == break_op; +} + static inline bool is_pc_ins(union loongarch_instruction *ip) { return ip->reg1i20_format.opcode >= pcaddi_op && diff --git a/arch/loongarch/include/asm/kprobes.h b/arch/loongarch/include/asm/kprobes.h new file mode 100644 index 000000000000..7b9fc3ed71c3 --- /dev/null +++ b/arch/loongarch/include/asm/kprobes.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_LOONGARCH_KPROBES_H +#define __ASM_LOONGARCH_KPROBES_H + +#include + +#ifdef CONFIG_KPROBES + +#include +#include + +#define __ARCH_WANT_KPROBES_INSN_SLOT +#define MAX_INSN_SIZE 2 + +#define flush_insn_slot(p) \ +do { \ + if (p->addr) \ + flush_icache_range((unsigned long)p->addr, \ + (unsigned long)p->addr + \ + (MAX_INSN_SIZE * sizeof(kprobe_opcode_t))); \ +} while (0) + +#define kretprobe_blacklist_size 0 + +typedef union loongarch_instruction kprobe_opcode_t; + +/* Architecture specific copy of original instruction */ +struct arch_specific_insn { + /* copy of the original instruction */ + kprobe_opcode_t *insn; + /* restore address after simulation */ + unsigned long restore; +}; + +struct prev_kprobe { + struct kprobe *kp; + unsigned int status; +}; + +/* per-cpu kprobe control block */ +struct kprobe_ctlblk { + unsigned int kprobe_status; + unsigned long saved_status; + struct prev_kprobe prev_kprobe; +}; + +void arch_remove_kprobe(struct kprobe *p); +bool kprobe_fault_handler(struct pt_regs *regs, int trapnr); +bool kprobe_breakpoint_handler(struct pt_regs *regs); +bool kprobe_singlestep_handler(struct pt_regs *regs); + +#else /* !CONFIG_KPROBES */ + +static inline bool kprobe_breakpoint_handler(struct pt_regs *regs) { return false; } +static inline bool kprobe_singlestep_handler(struct pt_regs *regs) { return false; } + +#endif /* CONFIG_KPROBES */ +#endif /* __ASM_LOONGARCH_KPROBES_H */ diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index bbb6b047f5db..0a2f546216c0 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -52,4 +52,6 @@ obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o +obj-$(CONFIG_KPROBES) += kprobes.o + CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS) diff --git a/arch/loongarch/kernel/kprobes.c b/arch/loongarch/kernel/kprobes.c new file mode 100644 index 000000000000..73eddd76f37c --- /dev/null +++ b/arch/loongarch/kernel/kprobes.c @@ -0,0 +1,379 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include + +static const union loongarch_instruction breakpoint_insn = { + .reg0i15_format = { + .opcode = break_op, + .immediate = BRK_KPROBE_BP, + } +}; + +static const union loongarch_instruction singlestep_insn = { + .reg0i15_format = { + .opcode = break_op, + .immediate = BRK_KPROBE_SSTEPBP, + } +}; + +DEFINE_PER_CPU(struct kprobe *, current_kprobe); +DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); + +static bool insns_not_supported(union loongarch_instruction insn) +{ + switch (insn.reg2i14_format.opcode) { + case llw_op: + case lld_op: + case scw_op: + case scd_op: + pr_notice("kprobe: ll and sc instructions are not supported\n"); + return true; + } + + switch (insn.reg1i21_format.opcode) { + case bceqz_op: + pr_notice("kprobe: bceqz and bcnez instructions are not supported\n"); + return true; + } + + return false; +} +NOKPROBE_SYMBOL(insns_not_supported); + +static bool insns_need_simulation(struct kprobe *p) +{ + if (is_pc_ins(&p->opcode)) + return true; + + if (is_branch_ins(&p->opcode)) + return true; + + return false; +} +NOKPROBE_SYMBOL(insns_need_simulation); + +static void arch_simulate_insn(struct kprobe *p, struct pt_regs *regs) +{ + if (is_pc_ins(&p->opcode)) + simu_pc(regs, p->opcode); + else if (is_branch_ins(&p->opcode)) + simu_branch(regs, p->opcode); +} +NOKPROBE_SYMBOL(arch_simulate_insn); + +static void arch_prepare_ss_slot(struct kprobe *p) +{ + p->ainsn.insn[0] = *p->addr; + p->ainsn.insn[1] = singlestep_insn; + p->ainsn.restore = (unsigned long)p->addr + LOONGARCH_INSN_SIZE; +} +NOKPROBE_SYMBOL(arch_prepare_ss_slot); + +static void arch_prepare_simulate(struct kprobe *p) +{ + p->ainsn.restore = 0; +} +NOKPROBE_SYMBOL(arch_prepare_simulate); + +int arch_prepare_kprobe(struct kprobe *p) +{ + if ((unsigned long)p->addr & 0x3) + return -EILSEQ; + + /* copy instruction */ + p->opcode = *p->addr; + + /* decode instruction */ + if (insns_not_supported(p->opcode)) + return -EINVAL; + + if (insns_need_simulation(p)) { + p->ainsn.insn = NULL; + } else { + p->ainsn.insn = get_insn_slot(); + if (!p->ainsn.insn) + return -ENOMEM; + } + + /* prepare the instruction */ + if (p->ainsn.insn) + arch_prepare_ss_slot(p); + else + arch_prepare_simulate(p); + + return 0; +} +NOKPROBE_SYMBOL(arch_prepare_kprobe); + +/* Install breakpoint in text */ +void arch_arm_kprobe(struct kprobe *p) +{ + *p->addr = breakpoint_insn; + flush_insn_slot(p); +} +NOKPROBE_SYMBOL(arch_arm_kprobe); + +/* Remove breakpoint from text */ +void arch_disarm_kprobe(struct kprobe *p) +{ + *p->addr = p->opcode; + flush_insn_slot(p); +} +NOKPROBE_SYMBOL(arch_disarm_kprobe); + +void arch_remove_kprobe(struct kprobe *p) +{ + if (p->ainsn.insn) { + free_insn_slot(p->ainsn.insn, 0); + p->ainsn.insn = NULL; + } +} +NOKPROBE_SYMBOL(arch_remove_kprobe); + +static void save_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + kcb->prev_kprobe.kp = kprobe_running(); + kcb->prev_kprobe.status = kcb->kprobe_status; +} +NOKPROBE_SYMBOL(save_previous_kprobe); + +static void restore_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); + kcb->kprobe_status = kcb->prev_kprobe.status; +} +NOKPROBE_SYMBOL(restore_previous_kprobe); + +static void set_current_kprobe(struct kprobe *p) +{ + __this_cpu_write(current_kprobe, p); +} +NOKPROBE_SYMBOL(set_current_kprobe); + +/* + * Interrupts need to be disabled before single-step mode is set, + * and not reenabled until after single-step mode ends. + * Without disabling interrupt on local CPU, there is a chance of + * interrupt occurrence in the period of exception return and start + * of out-of-line single-step, that result in wrongly single stepping + * into the interrupt handler. + */ +static void save_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) +{ + kcb->saved_status = regs->csr_prmd; + regs->csr_prmd &= ~CSR_PRMD_PIE; +} +NOKPROBE_SYMBOL(save_local_irqflag); + +static void restore_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) +{ + regs->csr_prmd = kcb->saved_status; +} +NOKPROBE_SYMBOL(restore_local_irqflag); + +static void post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, + struct pt_regs *regs) +{ + /* return addr restore if non-branching insn */ + if (cur->ainsn.restore != 0) + instruction_pointer_set(regs, cur->ainsn.restore); + + /* restore back original saved kprobe variables and continue */ + if (kcb->kprobe_status == KPROBE_REENTER) { + restore_previous_kprobe(kcb); + preempt_enable_no_resched(); + return; + } + + /* + * update the kcb status even if the cur->post_handler is + * not set because reset_curent_kprobe() doesn't update kcb. + */ + kcb->kprobe_status = KPROBE_HIT_SSDONE; + if (cur->post_handler) + cur->post_handler(cur, regs, 0); + + reset_current_kprobe(); + preempt_enable_no_resched(); +} +NOKPROBE_SYMBOL(post_kprobe_handler); + +static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb, int reenter) +{ + if (reenter) { + save_previous_kprobe(kcb); + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_REENTER; + } else { + kcb->kprobe_status = KPROBE_HIT_SS; + } + + if (p->ainsn.insn) { + /* IRQs and single stepping do not mix well */ + save_local_irqflag(kcb, regs); + /* set ip register to prepare for single stepping */ + regs->csr_era = (unsigned long)p->ainsn.insn; + } else { + /* simulate single steping */ + arch_simulate_insn(p, regs); + /* now go for post processing */ + post_kprobe_handler(p, kcb, regs); + } +} +NOKPROBE_SYMBOL(setup_singlestep); + +static bool reenter_kprobe(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + switch (kcb->kprobe_status) { + case KPROBE_HIT_SS: + case KPROBE_HIT_SSDONE: + case KPROBE_HIT_ACTIVE: + kprobes_inc_nmissed_count(p); + setup_singlestep(p, regs, kcb, 1); + break; + case KPROBE_REENTER: + pr_warn("Failed to recover from reentered kprobes.\n"); + dump_kprobe(p); + WARN_ON_ONCE(1); + break; + default: + WARN_ON(1); + return false; + } + + return true; +} +NOKPROBE_SYMBOL(reenter_kprobe); + +bool kprobe_breakpoint_handler(struct pt_regs *regs) +{ + struct kprobe_ctlblk *kcb; + struct kprobe *p, *cur_kprobe; + kprobe_opcode_t *addr = (kprobe_opcode_t *)regs->csr_era; + + /* + * We don't want to be preempted for the entire + * duration of kprobe processing. + */ + preempt_disable(); + kcb = get_kprobe_ctlblk(); + cur_kprobe = kprobe_running(); + + p = get_kprobe(addr); + if (p) { + if (cur_kprobe) { + if (reenter_kprobe(p, regs, kcb)) + return true; + } else { + /* Probe hit */ + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + + /* + * If we have no pre-handler or it returned 0, we + * continue with normal processing. If we have a + * pre-handler and it returned non-zero, it will + * modify the execution path and no need to single + * stepping. Let's just reset current kprobe and exit. + * + * pre_handler can hit a breakpoint and can step thru + * before return. + */ + if (!p->pre_handler || !p->pre_handler(p, regs)) { + setup_singlestep(p, regs, kcb, 0); + } else { + reset_current_kprobe(); + preempt_enable_no_resched(); + } + return true; + } + } + + if (addr->word != breakpoint_insn.word) { + /* + * The breakpoint instruction was removed right + * after we hit it. Another cpu has removed + * either a probepoint or a debugger breakpoint + * at this address. In either case, no further + * handling of this interrupt is appropriate. + * Return back to original instruction, and continue. + */ + regs->csr_era = (unsigned long)addr; + preempt_enable_no_resched(); + return true; + } + + preempt_enable_no_resched(); + return false; +} +NOKPROBE_SYMBOL(kprobe_breakpoint_handler); + +bool kprobe_singlestep_handler(struct pt_regs *regs) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + unsigned long addr = instruction_pointer(regs); + + if (cur && (kcb->kprobe_status & (KPROBE_HIT_SS | KPROBE_REENTER)) && + ((unsigned long)&cur->ainsn.insn[1] == addr)) { + restore_local_irqflag(kcb, regs); + post_kprobe_handler(cur, kcb, regs); + return true; + } + + preempt_enable_no_resched(); + return false; +} +NOKPROBE_SYMBOL(kprobe_singlestep_handler); + +bool kprobe_fault_handler(struct pt_regs *regs, int trapnr) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + switch (kcb->kprobe_status) { + case KPROBE_HIT_SS: + case KPROBE_REENTER: + /* + * We are here because the instruction being single + * stepped caused a page fault. We reset the current + * kprobe and the ip points back to the probe address + * and allow the page fault handler to continue as a + * normal page fault. + */ + regs->csr_era = (unsigned long)cur->addr; + WARN_ON_ONCE(!instruction_pointer(regs)); + + if (kcb->kprobe_status == KPROBE_REENTER) { + restore_previous_kprobe(kcb); + } else { + restore_local_irqflag(kcb, regs); + reset_current_kprobe(); + } + preempt_enable_no_resched(); + break; + } + return false; +} +NOKPROBE_SYMBOL(kprobe_fault_handler); + +/* + * Provide a blacklist of symbols identifying ranges which cannot be kprobed. + * This blacklist is exposed to userspace via debugfs (kprobes/blacklist). + */ +int __init arch_populate_kprobe_blacklist(void) +{ + return kprobe_add_area_blacklist((unsigned long)__irqentry_text_start, + (unsigned long)__irqentry_text_end); +} + +int __init arch_init_kprobes(void) +{ + return 0; +} diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c index 1d47747e2154..de8ebe20b666 100644 --- a/arch/loongarch/kernel/traps.c +++ b/arch/loongarch/kernel/traps.c @@ -437,7 +437,9 @@ asmlinkage void noinstr do_bp(struct pt_regs *regs) unsigned long era = exception_era(regs); irqentry_state_t state = irqentry_enter(regs); - local_irq_enable(); + if (regs->csr_prmd & CSR_PRMD_PIE) + local_irq_enable(); + current->thread.trap_nr = read_csr_excode(); if (__get_inst(&opcode, (u32 *)era, user)) goto out_sigsegv; @@ -450,14 +452,12 @@ asmlinkage void noinstr do_bp(struct pt_regs *regs) */ switch (bcode) { case BRK_KPROBE_BP: - if (notify_die(DIE_BREAK, "Kprobe", regs, bcode, - current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + if (kprobe_breakpoint_handler(regs)) goto out; else break; case BRK_KPROBE_SSTEPBP: - if (notify_die(DIE_SSTEPBP, "Kprobe_SingleStep", regs, bcode, - current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + if (kprobe_singlestep_handler(regs)) goto out; else break; @@ -500,7 +500,9 @@ asmlinkage void noinstr do_bp(struct pt_regs *regs) } out: - local_irq_disable(); + if (regs->csr_prmd & CSR_PRMD_PIE) + local_irq_disable(); + irqentry_exit(regs, state); return; diff --git a/arch/loongarch/mm/fault.c b/arch/loongarch/mm/fault.c index 1ccd53655cab..449087bd589d 100644 --- a/arch/loongarch/mm/fault.c +++ b/arch/loongarch/mm/fault.c @@ -135,6 +135,9 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, struct vm_area_struct *vma = NULL; vm_fault_t fault; + if (kprobe_page_fault(regs, current->thread.trap_nr)) + return; + /* * We fault-in kernel-space virtual memory on-demand. The * 'reference' page table is init_mm.pgd. -- cgit v1.2.3 From 3f5536860086d906b01ec5ed68cf50c7edcc40af Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Sat, 25 Feb 2023 15:52:57 +0800 Subject: LoongArch: Add kretprobes support Use the generic kretprobe trampoline handler to add kretprobes support for LoongArch. Tested-by: Jeff Xie Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/kprobes.h | 3 + arch/loongarch/kernel/Makefile | 2 +- arch/loongarch/kernel/kprobes.c | 27 +++++++++ arch/loongarch/kernel/kprobes_trampoline.S | 96 ++++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 arch/loongarch/kernel/kprobes_trampoline.S (limited to 'arch/loongarch') diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index bb538b68116d..d90bb47f1dc3 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -107,6 +107,7 @@ config LOONGARCH select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_IRQ_TIME_ACCOUNTING select HAVE_KPROBES + select HAVE_KRETPROBES select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI select HAVE_PCI diff --git a/arch/loongarch/include/asm/kprobes.h b/arch/loongarch/include/asm/kprobes.h index 7b9fc3ed71c3..798020ae02c6 100644 --- a/arch/loongarch/include/asm/kprobes.h +++ b/arch/loongarch/include/asm/kprobes.h @@ -49,6 +49,9 @@ bool kprobe_fault_handler(struct pt_regs *regs, int trapnr); bool kprobe_breakpoint_handler(struct pt_regs *regs); bool kprobe_singlestep_handler(struct pt_regs *regs); +void __kretprobe_trampoline(void); +void *trampoline_probe_handler(struct pt_regs *regs); + #else /* !CONFIG_KPROBES */ static inline bool kprobe_breakpoint_handler(struct pt_regs *regs) { return false; } diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index 0a2f546216c0..78d4e3384305 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -52,6 +52,6 @@ obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o -obj-$(CONFIG_KPROBES) += kprobes.o +obj-$(CONFIG_KPROBES) += kprobes.o kprobes_trampoline.o CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS) diff --git a/arch/loongarch/kernel/kprobes.c b/arch/loongarch/kernel/kprobes.c index 73eddd76f37c..56c8c4b09a42 100644 --- a/arch/loongarch/kernel/kprobes.c +++ b/arch/loongarch/kernel/kprobes.c @@ -377,3 +377,30 @@ int __init arch_init_kprobes(void) { return 0; } + +/* ASM function that handles the kretprobes must not be probed */ +NOKPROBE_SYMBOL(__kretprobe_trampoline); + +/* Called from __kretprobe_trampoline */ +void __used *trampoline_probe_handler(struct pt_regs *regs) +{ + return (void *)kretprobe_trampoline_handler(regs, NULL); +} +NOKPROBE_SYMBOL(trampoline_probe_handler); + +void arch_prepare_kretprobe(struct kretprobe_instance *ri, + struct pt_regs *regs) +{ + ri->ret_addr = (kprobe_opcode_t *)regs->regs[1]; + ri->fp = NULL; + + /* Replace the return addr with trampoline addr */ + regs->regs[1] = (unsigned long)&__kretprobe_trampoline; +} +NOKPROBE_SYMBOL(arch_prepare_kretprobe); + +int arch_trampoline_kprobe(struct kprobe *p) +{ + return 0; +} +NOKPROBE_SYMBOL(arch_trampoline_kprobe); diff --git a/arch/loongarch/kernel/kprobes_trampoline.S b/arch/loongarch/kernel/kprobes_trampoline.S new file mode 100644 index 000000000000..af94b0d213fa --- /dev/null +++ b/arch/loongarch/kernel/kprobes_trampoline.S @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#include +#include + + .text + + .macro save_all_base_regs + cfi_st ra, PT_R1 + cfi_st tp, PT_R2 + cfi_st a0, PT_R4 + cfi_st a1, PT_R5 + cfi_st a2, PT_R6 + cfi_st a3, PT_R7 + cfi_st a4, PT_R8 + cfi_st a5, PT_R9 + cfi_st a6, PT_R10 + cfi_st a7, PT_R11 + cfi_st t0, PT_R12 + cfi_st t1, PT_R13 + cfi_st t2, PT_R14 + cfi_st t3, PT_R15 + cfi_st t4, PT_R16 + cfi_st t5, PT_R17 + cfi_st t6, PT_R18 + cfi_st t7, PT_R19 + cfi_st t8, PT_R20 + cfi_st u0, PT_R21 + cfi_st fp, PT_R22 + cfi_st s0, PT_R23 + cfi_st s1, PT_R24 + cfi_st s2, PT_R25 + cfi_st s3, PT_R26 + cfi_st s4, PT_R27 + cfi_st s5, PT_R28 + cfi_st s6, PT_R29 + cfi_st s7, PT_R30 + cfi_st s8, PT_R31 + csrrd t0, LOONGARCH_CSR_CRMD + andi t0, t0, 0x7 /* extract bit[1:0] PLV, bit[2] IE */ + LONG_S t0, sp, PT_CRMD + .endm + + .macro restore_all_base_regs + cfi_ld tp, PT_R2 + cfi_ld a0, PT_R4 + cfi_ld a1, PT_R5 + cfi_ld a2, PT_R6 + cfi_ld a3, PT_R7 + cfi_ld a4, PT_R8 + cfi_ld a5, PT_R9 + cfi_ld a6, PT_R10 + cfi_ld a7, PT_R11 + cfi_ld t0, PT_R12 + cfi_ld t1, PT_R13 + cfi_ld t2, PT_R14 + cfi_ld t3, PT_R15 + cfi_ld t4, PT_R16 + cfi_ld t5, PT_R17 + cfi_ld t6, PT_R18 + cfi_ld t7, PT_R19 + cfi_ld t8, PT_R20 + cfi_ld u0, PT_R21 + cfi_ld fp, PT_R22 + cfi_ld s0, PT_R23 + cfi_ld s1, PT_R24 + cfi_ld s2, PT_R25 + cfi_ld s3, PT_R26 + cfi_ld s4, PT_R27 + cfi_ld s5, PT_R28 + cfi_ld s6, PT_R29 + cfi_ld s7, PT_R30 + cfi_ld s8, PT_R31 + LONG_L t0, sp, PT_CRMD + li.d t1, 0x7 /* mask bit[1:0] PLV, bit[2] IE */ + csrxchg t0, t1, LOONGARCH_CSR_CRMD + .endm + +SYM_CODE_START(__kretprobe_trampoline) + addi.d sp, sp, -PT_SIZE + save_all_base_regs + + addi.d t0, sp, PT_SIZE + LONG_S t0, sp, PT_R3 + + move a0, sp /* pt_regs */ + + bl trampoline_probe_handler + + /* use the result as the return-address */ + move ra, a0 + + restore_all_base_regs + addi.d sp, sp, PT_SIZE + + jr ra +SYM_CODE_END(__kretprobe_trampoline) -- cgit v1.2.3 From 09e679c28a4def4ea8d4f618503c1d55c355ae5c Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Sat, 25 Feb 2023 15:52:57 +0800 Subject: LoongArch: Add kprobes on ftrace support Add kprobe_ftrace_handler() and arch_prepare_kprobe_ftrace() to support kprobes on ftrace, the code is similar with x86 and riscv. Here is a simple example: # echo 'p:myprobe kernel_clone' > /sys/kernel/debug/tracing/kprobe_events # echo 'r:myretprobe kernel_clone $retval' >> /sys/kernel/debug/tracing/kprobe_events # echo 1 > /sys/kernel/debug/tracing/events/kprobes/myprobe/enable # echo 1 > /sys/kernel/debug/tracing/events/kprobes/myretprobe/enable # echo 1 > /sys/kernel/debug/tracing/tracing_on # cat /sys/kernel/debug/tracing/trace # tracer: nop # # entries-in-buffer/entries-written: 2/2 #P:4 # # _-----=> irqs-off/BH-disabled # / _----=> need-resched # | / _---=> hardirq/softirq # || / _--=> preempt-depth # ||| / _-=> migrate-disable # |||| / delay # TASK-PID CPU# ||||| TIMESTAMP FUNCTION # | | | ||||| | | bash-488 [002] ..... 2041.190681: myprobe: (kernel_clone+0x0/0x40c) bash-488 [002] ..... 2041.190788: myretprobe: (__do_sys_clone+0x84/0xb8 <- kernel_clone) arg1=0x200 Tested-by: Jeff Xie Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 1 + arch/loongarch/kernel/ftrace_dyn.c | 64 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) (limited to 'arch/loongarch') diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index d90bb47f1dc3..7fd51257e0ed 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -107,6 +107,7 @@ config LOONGARCH select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_IRQ_TIME_ACCOUNTING select HAVE_KPROBES + select HAVE_KPROBES_ON_FTRACE select HAVE_KRETPROBES select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI diff --git a/arch/loongarch/kernel/ftrace_dyn.c b/arch/loongarch/kernel/ftrace_dyn.c index 0f07591cab30..4a3ef8516ccc 100644 --- a/arch/loongarch/kernel/ftrace_dyn.c +++ b/arch/loongarch/kernel/ftrace_dyn.c @@ -6,6 +6,7 @@ */ #include +#include #include #include @@ -271,3 +272,66 @@ int ftrace_disable_ftrace_graph_caller(void) } #endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */ #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ + +#ifdef CONFIG_KPROBES_ON_FTRACE +/* Ftrace callback handler for kprobes -- called under preepmt disabled */ +void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *ops, struct ftrace_regs *fregs) +{ + int bit; + struct pt_regs *regs; + struct kprobe *p; + struct kprobe_ctlblk *kcb; + + bit = ftrace_test_recursion_trylock(ip, parent_ip); + if (bit < 0) + return; + + p = get_kprobe((kprobe_opcode_t *)ip); + if (unlikely(!p) || kprobe_disabled(p)) + goto out; + + regs = ftrace_get_regs(fregs); + if (!regs) + goto out; + + kcb = get_kprobe_ctlblk(); + if (kprobe_running()) { + kprobes_inc_nmissed_count(p); + } else { + unsigned long orig_ip = instruction_pointer(regs); + + instruction_pointer_set(regs, ip); + + __this_cpu_write(current_kprobe, p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + if (!p->pre_handler || !p->pre_handler(p, regs)) { + /* + * Emulate singlestep (and also recover regs->csr_era) + * as if there is a nop + */ + instruction_pointer_set(regs, (unsigned long)p->addr + MCOUNT_INSN_SIZE); + if (unlikely(p->post_handler)) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + p->post_handler(p, regs, 0); + } + instruction_pointer_set(regs, orig_ip); + } + + /* + * If pre_handler returns !0, it changes regs->csr_era. We have to + * skip emulating post_handler. + */ + __this_cpu_write(current_kprobe, NULL); + } +out: + ftrace_test_recursion_unlock(bit); +} +NOKPROBE_SYMBOL(kprobe_ftrace_handler); + +int arch_prepare_kprobe_ftrace(struct kprobe *p) +{ + p->ainsn.insn = NULL; + return 0; +} +#endif /* CONFIG_KPROBES_ON_FTRACE */ -- cgit v1.2.3 From fcf77d016216ae75a19078e1079c2511a6319a49 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Sat, 25 Feb 2023 15:52:57 +0800 Subject: LoongArch: Mark some assembler symbols as non-kprobe-able Some assembler symbols are not kprobe safe, such as handle_syscall (used as syscall exception handler), *memset*/*memcpy*/*memmove* (may cause recursive exceptions), they can not be instrumented, just blacklist them for kprobing. Here is a related problem and discussion: Link: https://lore.kernel.org/lkml/20230114143859.7ccc45c1c5d9ce302113ab0a@kernel.org/ Tested-by: Jeff Xie Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/asm.h | 10 ++++++++++ arch/loongarch/kernel/entry.S | 1 + arch/loongarch/lib/memcpy.S | 3 +++ arch/loongarch/lib/memmove.S | 4 ++++ arch/loongarch/lib/memset.S | 3 +++ 5 files changed, 21 insertions(+) (limited to 'arch/loongarch') diff --git a/arch/loongarch/include/asm/asm.h b/arch/loongarch/include/asm/asm.h index 40eea6aa469e..f591b3245def 100644 --- a/arch/loongarch/include/asm/asm.h +++ b/arch/loongarch/include/asm/asm.h @@ -188,4 +188,14 @@ #define PTRLOG 3 #endif +/* Annotate a function as being unsuitable for kprobes. */ +#ifdef CONFIG_KPROBES +#define _ASM_NOKPROBE(name) \ + .pushsection "_kprobe_blacklist", "aw"; \ + .quad name; \ + .popsection +#else +#define _ASM_NOKPROBE(name) +#endif + #endif /* __ASM_ASM_H */ diff --git a/arch/loongarch/kernel/entry.S b/arch/loongarch/kernel/entry.S index f14ab6a0e0b0..d737e3cf42d3 100644 --- a/arch/loongarch/kernel/entry.S +++ b/arch/loongarch/kernel/entry.S @@ -67,6 +67,7 @@ SYM_FUNC_START(handle_syscall) RESTORE_ALL_AND_RET SYM_FUNC_END(handle_syscall) +_ASM_NOKPROBE(handle_syscall) SYM_CODE_START(ret_from_fork) bl schedule_tail # a0 = struct task_struct *prev diff --git a/arch/loongarch/lib/memcpy.S b/arch/loongarch/lib/memcpy.S index 7c07d595ee89..3b7e1dec7109 100644 --- a/arch/loongarch/lib/memcpy.S +++ b/arch/loongarch/lib/memcpy.S @@ -17,6 +17,7 @@ SYM_FUNC_START(memcpy) ALTERNATIVE "b __memcpy_generic", \ "b __memcpy_fast", CPU_FEATURE_UAL SYM_FUNC_END(memcpy) +_ASM_NOKPROBE(memcpy) EXPORT_SYMBOL(memcpy) @@ -41,6 +42,7 @@ SYM_FUNC_START(__memcpy_generic) 2: move a0, a3 jr ra SYM_FUNC_END(__memcpy_generic) +_ASM_NOKPROBE(__memcpy_generic) /* * void *__memcpy_fast(void *dst, const void *src, size_t n) @@ -93,3 +95,4 @@ SYM_FUNC_START(__memcpy_fast) 3: move a0, a3 jr ra SYM_FUNC_END(__memcpy_fast) +_ASM_NOKPROBE(__memcpy_fast) diff --git a/arch/loongarch/lib/memmove.S b/arch/loongarch/lib/memmove.S index 6ffdb46da78f..b796c3d6da05 100644 --- a/arch/loongarch/lib/memmove.S +++ b/arch/loongarch/lib/memmove.S @@ -29,6 +29,7 @@ SYM_FUNC_START(memmove) b rmemcpy 4: b __rmemcpy_generic SYM_FUNC_END(memmove) +_ASM_NOKPROBE(memmove) EXPORT_SYMBOL(memmove) @@ -39,6 +40,7 @@ SYM_FUNC_START(rmemcpy) ALTERNATIVE "b __rmemcpy_generic", \ "b __rmemcpy_fast", CPU_FEATURE_UAL SYM_FUNC_END(rmemcpy) +_ASM_NOKPROBE(rmemcpy) /* * void *__rmemcpy_generic(void *dst, const void *src, size_t n) @@ -64,6 +66,7 @@ SYM_FUNC_START(__rmemcpy_generic) 2: move a0, a3 jr ra SYM_FUNC_END(__rmemcpy_generic) +_ASM_NOKPROBE(__rmemcpy_generic) /* * void *__rmemcpy_fast(void *dst, const void *src, size_t n) @@ -119,3 +122,4 @@ SYM_FUNC_START(__rmemcpy_fast) 3: move a0, a3 jr ra SYM_FUNC_END(__rmemcpy_fast) +_ASM_NOKPROBE(__rmemcpy_fast) diff --git a/arch/loongarch/lib/memset.S b/arch/loongarch/lib/memset.S index e7cb4ea3747d..a9eb732ab2ad 100644 --- a/arch/loongarch/lib/memset.S +++ b/arch/loongarch/lib/memset.S @@ -23,6 +23,7 @@ SYM_FUNC_START(memset) ALTERNATIVE "b __memset_generic", \ "b __memset_fast", CPU_FEATURE_UAL SYM_FUNC_END(memset) +_ASM_NOKPROBE(memset) EXPORT_SYMBOL(memset) @@ -45,6 +46,7 @@ SYM_FUNC_START(__memset_generic) 2: move a0, a3 jr ra SYM_FUNC_END(__memset_generic) +_ASM_NOKPROBE(__memset_generic) /* * void *__memset_fast(void *s, int c, size_t n) @@ -89,3 +91,4 @@ SYM_FUNC_START(__memset_fast) 3: move a0, a3 jr ra SYM_FUNC_END(__memset_fast) +_ASM_NOKPROBE(__memset_fast) -- cgit v1.2.3