From bd52cd5e23f134019b23f0c389db0f9a436e4576 Mon Sep 17 00:00:00 2001 From: Jann Horn Date: Tue, 17 May 2022 16:30:47 +0200 Subject: s390/crypto: fix scatterwalk_unmap() callers in AES-GCM The argument of scatterwalk_unmap() is supposed to be the void* that was returned by the previous scatterwalk_map() call. The s390 AES-GCM implementation was instead passing the pointer to the struct scatter_walk. This doesn't actually break anything because scatterwalk_unmap() only uses its argument under CONFIG_HIGHMEM and ARCH_HAS_FLUSH_ON_KUNMAP. Fixes: bf7fa038707c ("s390/crypto: add s390 platform specific aes gcm support.") Signed-off-by: Jann Horn Acked-by: Harald Freudenberger Link: https://lore.kernel.org/r/20220517143047.3054498-1-jannh@google.com Signed-off-by: Heiko Carstens --- arch/s390/crypto/aes_s390.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/s390') diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 54c7536f2482..1023e9d43d44 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -701,7 +701,7 @@ static inline void _gcm_sg_unmap_and_advance(struct gcm_sg_walk *gw, unsigned int nbytes) { gw->walk_bytes_remain -= nbytes; - scatterwalk_unmap(&gw->walk); + scatterwalk_unmap(gw->walk_ptr); scatterwalk_advance(&gw->walk, nbytes); scatterwalk_done(&gw->walk, 0, gw->walk_bytes_remain); gw->walk_ptr = NULL; @@ -776,7 +776,7 @@ static int gcm_out_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded) goto out; } - scatterwalk_unmap(&gw->walk); + scatterwalk_unmap(gw->walk_ptr); gw->walk_ptr = NULL; gw->ptr = gw->buf; -- cgit v1.2.3 From 85806016acb0610e47ffcd5415d1575f18f58a33 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 20 May 2022 19:23:48 +0200 Subject: s390: simplify early program check handler Due to historic reasons the base program check handler calls a configurable function. Given that there is only the early program check handler left, simplify the code by directly calling that function. The only other user was removed with commit d485235b0054 ("s390: assume diag308 set always works"). Also rename all functions and the asm file to reflect this. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/include/asm/processor.h | 6 ----- arch/s390/kernel/Makefile | 2 +- arch/s390/kernel/base.S | 50 --------------------------------------- arch/s390/kernel/early.c | 5 ++-- arch/s390/kernel/earlypgm.S | 26 ++++++++++++++++++++ arch/s390/kernel/entry.h | 2 ++ 6 files changed, 31 insertions(+), 60 deletions(-) delete mode 100644 arch/s390/kernel/base.S create mode 100644 arch/s390/kernel/earlypgm.S (limited to 'arch/s390') diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index add764a2be8c..bd66f8e34949 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -304,12 +304,6 @@ static __always_inline void __noreturn disabled_wait(void) while (1); } -/* - * Basic Program Check Handler. - */ -extern void s390_base_pgm_handler(void); -extern void (*s390_base_pgm_handler_fn)(struct pt_regs *regs); - #define ARCH_LOW_ADDRESS_LIMIT 0x7fffffffUL extern int memcpy_real(void *, unsigned long, size_t); diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 5851041bb214..27d6b3c7aa06 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -33,7 +33,7 @@ CFLAGS_stacktrace.o += -fno-optimize-sibling-calls CFLAGS_dumpstack.o += -fno-optimize-sibling-calls CFLAGS_unwind_bc.o += -fno-optimize-sibling-calls -obj-y := traps.o time.o process.o base.o early.o setup.o idle.o vtime.o +obj-y := traps.o time.o process.o earlypgm.o early.o setup.o idle.o vtime.o obj-y += processor.o syscall.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o obj-y += sysinfo.o lgr.o os_info.o machine_kexec.o diff --git a/arch/s390/kernel/base.S b/arch/s390/kernel/base.S deleted file mode 100644 index 172c23c8ca00..000000000000 --- a/arch/s390/kernel/base.S +++ /dev/null @@ -1,50 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * arch/s390/kernel/base.S - * - * Copyright IBM Corp. 2006, 2007 - * Author(s): Michael Holzheu - */ - -#include -#include -#include -#include - - GEN_BR_THUNK %r9 - GEN_BR_THUNK %r14 - -__PT_R0 = __PT_GPRS -__PT_R8 = __PT_GPRS + 64 - -ENTRY(s390_base_pgm_handler) - stmg %r8,%r15,__LC_SAVE_AREA_SYNC - aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE) - la %r11,STACK_FRAME_OVERHEAD(%r15) - xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) - stmg %r0,%r7,__PT_R0(%r11) - mvc __PT_PSW(16,%r11),__LC_PGM_OLD_PSW - mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC - lgr %r2,%r11 - larl %r1,s390_base_pgm_handler_fn - lg %r9,0(%r1) - ltgr %r9,%r9 - jz 1f - BASR_EX %r14,%r9 - mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) - lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) - lpswe __LC_RETURN_PSW -1: larl %r13,disabled_wait_psw - lpswe 0(%r13) -ENDPROC(s390_base_pgm_handler) - - .align 8 -disabled_wait_psw: - .quad 0x0002000180000000,0x0000000000000000 + s390_base_pgm_handler - - .section .bss - .align 8 - .globl s390_base_pgm_handler_fn -s390_base_pgm_handler_fn: - .quad 0 - .previous diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 08cc86a0db90..432c8c987256 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -149,7 +149,7 @@ static __init void setup_topology(void) topology_max_mnest = max_mnest; } -static void early_pgm_check_handler(struct pt_regs *regs) +void __do_early_pgm_check(struct pt_regs *regs) { if (!fixup_exception(regs)) disabled_wait(); @@ -159,12 +159,11 @@ static noinline __init void setup_lowcore_early(void) { psw_t psw; - psw.addr = (unsigned long)s390_base_pgm_handler; + psw.addr = (unsigned long)early_pgm_check_handler; psw.mask = PSW_MASK_BASE | PSW_DEFAULT_KEY | PSW_MASK_EA | PSW_MASK_BA; if (IS_ENABLED(CONFIG_KASAN)) psw.mask |= PSW_MASK_DAT; S390_lowcore.program_new_psw = psw; - s390_base_pgm_handler_fn = early_pgm_check_handler; S390_lowcore.preempt_count = INIT_PREEMPT_COUNT; } diff --git a/arch/s390/kernel/earlypgm.S b/arch/s390/kernel/earlypgm.S new file mode 100644 index 000000000000..fbc7d2d538df --- /dev/null +++ b/arch/s390/kernel/earlypgm.S @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright IBM Corp. 2006, 2007 + * Author(s): Michael Holzheu + */ + +#include +#include + +__PT_R0 = __PT_GPRS +__PT_R8 = __PT_GPRS + 64 + +ENTRY(early_pgm_check_handler) + stmg %r8,%r15,__LC_SAVE_AREA_SYNC + aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE) + la %r11,STACK_FRAME_OVERHEAD(%r15) + xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) + stmg %r0,%r7,__PT_R0(%r11) + mvc __PT_PSW(16,%r11),__LC_PGM_OLD_PSW + mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC + lgr %r2,%r11 + brasl %r14,__do_early_pgm_check + mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) + lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) + lpswe __LC_RETURN_PSW +ENDPROC(early_pgm_check_handler) diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index 56e5e3712fbb..995ec7449feb 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -17,10 +17,12 @@ void ext_int_handler(void); void io_int_handler(void); void mcck_int_handler(void); void restart_int_handler(void); +void early_pgm_check_handler(void); void __ret_from_fork(struct task_struct *prev, struct pt_regs *regs); void __do_pgm_check(struct pt_regs *regs); void __do_syscall(struct pt_regs *regs, int per_trap); +void __do_early_pgm_check(struct pt_regs *regs); void do_protection_exception(struct pt_regs *regs); void do_dat_exception(struct pt_regs *regs); -- cgit v1.2.3 From 3384f135e90f96a7697a85c33081e46c0a0f415d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 23 May 2022 14:42:40 +0200 Subject: s390: generate register offsets into pt_regs automatically Use asm offsets method to generate register offsets into pt_regs, instead of open-coding at several places. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/kernel/asm-offsets.c | 16 ++++++++++++++++ arch/s390/kernel/earlypgm.S | 3 --- arch/s390/kernel/entry.S | 17 ----------------- 3 files changed, 16 insertions(+), 20 deletions(-) (limited to 'arch/s390') diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index 7c74f0e17e5a..1e37cad44561 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -32,6 +32,22 @@ int main(void) /* pt_regs offsets */ OFFSET(__PT_PSW, pt_regs, psw); OFFSET(__PT_GPRS, pt_regs, gprs); + OFFSET(__PT_R0, pt_regs, gprs[0]); + OFFSET(__PT_R1, pt_regs, gprs[1]); + OFFSET(__PT_R2, pt_regs, gprs[2]); + OFFSET(__PT_R3, pt_regs, gprs[3]); + OFFSET(__PT_R4, pt_regs, gprs[4]); + OFFSET(__PT_R5, pt_regs, gprs[5]); + OFFSET(__PT_R6, pt_regs, gprs[6]); + OFFSET(__PT_R7, pt_regs, gprs[7]); + OFFSET(__PT_R8, pt_regs, gprs[8]); + OFFSET(__PT_R9, pt_regs, gprs[9]); + OFFSET(__PT_R10, pt_regs, gprs[10]); + OFFSET(__PT_R11, pt_regs, gprs[11]); + OFFSET(__PT_R12, pt_regs, gprs[12]); + OFFSET(__PT_R13, pt_regs, gprs[13]); + OFFSET(__PT_R14, pt_regs, gprs[14]); + OFFSET(__PT_R15, pt_regs, gprs[15]); OFFSET(__PT_ORIG_GPR2, pt_regs, orig_gpr2); OFFSET(__PT_FLAGS, pt_regs, flags); OFFSET(__PT_CR1, pt_regs, cr1); diff --git a/arch/s390/kernel/earlypgm.S b/arch/s390/kernel/earlypgm.S index fbc7d2d538df..f521c6da37b8 100644 --- a/arch/s390/kernel/earlypgm.S +++ b/arch/s390/kernel/earlypgm.S @@ -7,9 +7,6 @@ #include #include -__PT_R0 = __PT_GPRS -__PT_R8 = __PT_GPRS + 64 - ENTRY(early_pgm_check_handler) stmg %r8,%r15,__LC_SAVE_AREA_SYNC aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE) diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index df41132ccd06..82faf4d0e4d7 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -29,23 +29,6 @@ #include #include -__PT_R0 = __PT_GPRS -__PT_R1 = __PT_GPRS + 8 -__PT_R2 = __PT_GPRS + 16 -__PT_R3 = __PT_GPRS + 24 -__PT_R4 = __PT_GPRS + 32 -__PT_R5 = __PT_GPRS + 40 -__PT_R6 = __PT_GPRS + 48 -__PT_R7 = __PT_GPRS + 56 -__PT_R8 = __PT_GPRS + 64 -__PT_R9 = __PT_GPRS + 72 -__PT_R10 = __PT_GPRS + 80 -__PT_R11 = __PT_GPRS + 88 -__PT_R12 = __PT_GPRS + 96 -__PT_R13 = __PT_GPRS + 104 -__PT_R14 = __PT_GPRS + 112 -__PT_R15 = __PT_GPRS + 120 - STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER STACK_SIZE = 1 << STACK_SHIFT STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE -- cgit v1.2.3 From c9bfb460c3e4da2462e16b0f0b200990b36b1dd2 Mon Sep 17 00:00:00 2001 From: Nico Boehr Date: Tue, 24 May 2022 15:43:20 +0200 Subject: s390/perf: obtain sie_block from the right address Since commit 1179f170b6f0 ("s390: fix fpu restore in entry.S"), the sie_block pointer is located at empty1[1], but in sie_block() it was taken from empty1[0]. This leads to a random pointer being dereferenced, possibly causing system crash. This problem can be observed when running a simple guest with an endless loop and recording the cpu-clock event: sudo perf kvm --guestvmlinux= --guest top -e cpu-clock With this fix, the correct guest address is shown. Fixes: 1179f170b6f0 ("s390: fix fpu restore in entry.S") Cc: stable@vger.kernel.org Acked-by: Christian Borntraeger Acked-by: Claudio Imbrenda Reviewed-by: Heiko Carstens Signed-off-by: Nico Boehr Signed-off-by: Heiko Carstens --- arch/s390/kernel/perf_event.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/s390') diff --git a/arch/s390/kernel/perf_event.c b/arch/s390/kernel/perf_event.c index ea7729bebaa0..a7f8db73984b 100644 --- a/arch/s390/kernel/perf_event.c +++ b/arch/s390/kernel/perf_event.c @@ -30,7 +30,7 @@ static struct kvm_s390_sie_block *sie_block(struct pt_regs *regs) if (!stack) return NULL; - return (struct kvm_s390_sie_block *) stack->empty1[0]; + return (struct kvm_s390_sie_block *)stack->empty1[1]; } static bool is_in_guest(struct pt_regs *regs) -- cgit v1.2.3 From 4d1f2e4576af57f8d6b357198dcdda88d0431d5e Mon Sep 17 00:00:00 2001 From: Juerg Haefliger Date: Wed, 25 May 2022 14:01:40 +0200 Subject: s390/Kconfig: fix indentation The convention for indentation seems to be a single tab. Help text is further indented by an additional two whitespaces. Fix the lines that violate these rules. Signed-off-by: Juerg Haefliger Link: https://lore.kernel.org/r/20220525120140.39534-1-juerg.haefliger@canonical.com Signed-off-by: Heiko Carstens --- arch/s390/Kconfig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'arch/s390') diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index e084c72104f8..543e859905df 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -736,11 +736,11 @@ config VFIO_AP depends on S390_AP_IOMMU && VFIO_MDEV && KVM depends on ZCRYPT help - This driver grants access to Adjunct Processor (AP) devices - via the VFIO mediated device interface. + This driver grants access to Adjunct Processor (AP) devices + via the VFIO mediated device interface. - To compile this driver as a module, choose M here: the module - will be called vfio_ap. + To compile this driver as a module, choose M here: the module + will be called vfio_ap. endmenu -- cgit v1.2.3 From 13e282cc92d7e4b96fad8b71adf48b7eab22f6b0 Mon Sep 17 00:00:00 2001 From: Juerg Haefliger Date: Wed, 25 May 2022 14:01:51 +0200 Subject: s390/Kconfig.debug: fix indentation The convention for indentation seems to be a single tab. Help text is further indented by an additional two whitespaces. Fix the lines that violate these rules. Signed-off-by: Juerg Haefliger Link: https://lore.kernel.org/r/20220525120151.39594-1-juerg.haefliger@canonical.com Signed-off-by: Heiko Carstens --- arch/s390/Kconfig.debug | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'arch/s390') diff --git a/arch/s390/Kconfig.debug b/arch/s390/Kconfig.debug index e94a2a7f6bf4..c4300ea4abf8 100644 --- a/arch/s390/Kconfig.debug +++ b/arch/s390/Kconfig.debug @@ -14,9 +14,9 @@ config DEBUG_ENTRY If unsure, say N. config CIO_INJECT - bool "CIO Inject interfaces" - depends on DEBUG_KERNEL && DEBUG_FS - help - This option provides a debugging facility to inject certain artificial events - and instruction responses to the CIO layer of Linux kernel. The newly created - debugfs user-interfaces will be at /sys/kernel/debug/s390/cio/* + bool "CIO Inject interfaces" + depends on DEBUG_KERNEL && DEBUG_FS + help + This option provides a debugging facility to inject certain artificial events + and instruction responses to the CIO layer of Linux kernel. The newly created + debugfs user-interfaces will be at /sys/kernel/debug/s390/cio/* -- cgit v1.2.3 From 9dade1707fdb4fb6a1d498aa8811779556b7ffbc Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 30 May 2022 11:37:48 +0200 Subject: s390/kexec: add __GFP_NORETRY to KEXEC_CONTROL_MEMORY_GFP Avoid invoking the OOM-killer when allocating the control page. This is the s390 variant of commit dc5cccacf427 ("kexec: don't invoke OOM-killer for control page allocation"). Signed-off-by: Heiko Carstens --- arch/s390/include/asm/kexec.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/s390') diff --git a/arch/s390/include/asm/kexec.h b/arch/s390/include/asm/kexec.h index 7f3c9ac34bd8..9df3b4d64724 100644 --- a/arch/s390/include/asm/kexec.h +++ b/arch/s390/include/asm/kexec.h @@ -29,7 +29,7 @@ #define KEXEC_CONTROL_MEMORY_LIMIT (1UL<<31) /* Allocate control page with GFP_DMA */ -#define KEXEC_CONTROL_MEMORY_GFP GFP_DMA +#define KEXEC_CONTROL_MEMORY_GFP (GFP_DMA | __GFP_NORETRY) /* Maximum address we can use for the crash control pages */ #define KEXEC_CRASH_CONTROL_MEMORY_LIMIT (-1UL) -- cgit v1.2.3 From 6d5946274df1fff539a7eece458a43be733d1db8 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Mon, 30 May 2022 11:27:05 +0200 Subject: s390/gmap: voluntarily schedule during key setting With large and many guest with storage keys it is possible to create large latencies or stalls during initial key setting: rcu: INFO: rcu_sched self-detected stall on CPU rcu: 18-....: (2099 ticks this GP) idle=54e/1/0x4000000000000002 softirq=35598716/35598716 fqs=998 (t=2100 jiffies g=155867385 q=20879) Task dump for CPU 18: CPU 1/KVM R running task 0 1030947 256019 0x06000004 Call Trace: sched_show_task rcu_dump_cpu_stacks rcu_sched_clock_irq update_process_times tick_sched_handle tick_sched_timer __hrtimer_run_queues hrtimer_interrupt do_IRQ ext_int_handler ptep_zap_key The mmap lock is held during the page walking but since this is a semaphore scheduling is still possible. Same for the kvm srcu. To minimize overhead do this on every segment table entry or large page. Signed-off-by: Christian Borntraeger Reviewed-by: Alexander Gordeev Reviewed-by: Claudio Imbrenda Link: https://lore.kernel.org/r/20220530092706.11637-2-borntraeger@linux.ibm.com Signed-off-by: Christian Borntraeger Signed-off-by: Heiko Carstens --- arch/s390/mm/gmap.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'arch/s390') diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index af03cacf34ec..f3694f6df8b7 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -2601,6 +2601,18 @@ static int __s390_enable_skey_pte(pte_t *pte, unsigned long addr, return 0; } +/* + * Give a chance to schedule after setting a key to 256 pages. + * We only hold the mm lock, which is a rwsem and the kvm srcu. + * Both can sleep. + */ +static int __s390_enable_skey_pmd(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + cond_resched(); + return 0; +} + static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr, unsigned long hmask, unsigned long next, struct mm_walk *walk) @@ -2623,12 +2635,14 @@ static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr, end = start + HPAGE_SIZE - 1; __storage_key_init_range(start, end); set_bit(PG_arch_1, &page->flags); + cond_resched(); return 0; } static const struct mm_walk_ops enable_skey_walk_ops = { .hugetlb_entry = __s390_enable_skey_hugetlb, .pte_entry = __s390_enable_skey_pte, + .pmd_entry = __s390_enable_skey_pmd, }; int s390_enable_skey(void) -- cgit v1.2.3 From 3ae11dbcfac906a8c3a480e98660a823130dc16a Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Mon, 30 May 2022 11:27:06 +0200 Subject: s390/mm: use non-quiescing sske for KVM switch to keyed guest The switch to a keyed guest does not require a classic sske as the other guest CPUs are not accessing the key before the switch is complete. By using the NQ SSKE things are faster especially with multiple guests. Signed-off-by: Christian Borntraeger Suggested-by: Janis Schoetterl-Glausch Reviewed-by: Claudio Imbrenda Link: https://lore.kernel.org/r/20220530092706.11637-3-borntraeger@linux.ibm.com Signed-off-by: Christian Borntraeger Signed-off-by: Heiko Carstens --- arch/s390/mm/pgtable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/s390') diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 697df02362af..4909dcd762e8 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -748,7 +748,7 @@ void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) pgste_val(pgste) |= PGSTE_GR_BIT | PGSTE_GC_BIT; ptev = pte_val(*ptep); if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE)) - page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1); + page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 0); pgste_set_unlock(ptep, pgste); preempt_enable(); } -- cgit v1.2.3 From 29ccaa4b35ea874ddd50518e5c2c746b9238a792 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Mon, 23 May 2022 12:38:14 +0200 Subject: s390/mcck: isolate SIE instruction when setting CIF_MCCK_GUEST flag Commit d768bd892fc8 ("s390: add options to change branch prediction behaviour for the kernel") introduced .Lsie_exit label - supposedly to fence off SIE instruction. However, the corresponding address range length .Lsie_crit_mcck_length was not updated, which led to BPON code potentionally marked with CIF_MCCK_GUEST flag. Both .Lsie_exit and .Lsie_crit_mcck_length were removed with commit 0b0ed657fe00 ("s390: remove critical section cleanup from entry.S"), but the issue persisted - currently BPOFF and BPENTER macros might get wrongly considered by the machine check handler as a guest. Fixes: d768bd892fc8 ("s390: add options to change branch prediction behaviour for the kernel") Reviewed-by: Sven Schnelle Reviewed-by: Christian Borntraeger Signed-off-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/kernel/entry.S | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'arch/s390') diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 82faf4d0e4d7..d2a1f2f4f5b8 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -251,6 +251,10 @@ ENTRY(sie64a) BPEXIT __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) .Lsie_entry: sie 0(%r14) +# Let the next instruction be NOP to avoid triggering a machine check +# and handling it in a guest as result of the instruction execution. + nopr 7 +.Lsie_leave: BPOFF BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) .Lsie_skip: @@ -547,7 +551,7 @@ ENTRY(mcck_int_handler) jno .Lmcck_panic #if IS_ENABLED(CONFIG_KVM) OUTSIDE %r9,.Lsie_gmap,.Lsie_done,6f - OUTSIDE %r9,.Lsie_entry,.Lsie_skip,4f + OUTSIDE %r9,.Lsie_entry,.Lsie_leave,4f oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST j 5f 4: CHKSTG .Lmcck_panic -- cgit v1.2.3 From 79a74dac447be957178e8596706b1db3d5c714f4 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sun, 29 May 2022 18:55:06 +0200 Subject: s390/uaccess: use symbolic names for inline assembler operands Make code easier to read by using symbolic names. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/uaccess.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'arch/s390') diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 1f150a7cfb3d..5ffb88b3fdee 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -278,12 +278,12 @@ int __noreturn __put_kernel_bad(void); int __rc; \ \ asm volatile( \ - "0: " insn " %2,%1\n" \ - "1: xr %0,%0\n" \ + "0: " insn " %[_val],%[_to]\n" \ + "1: xr %[rc],%[rc]\n" \ "2:\n" \ EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0) \ - : "=d" (__rc), "+Q" (*(to)) \ - : "d" (val) \ + : [rc] "=d" (__rc), [_to] "+Q" (*(to)) \ + : [_val] "d" (val) \ : "cc"); \ __rc; \ }) @@ -321,12 +321,12 @@ int __noreturn __get_kernel_bad(void); int __rc; \ \ asm volatile( \ - "0: " insn " %1,%2\n" \ - "1: xr %0,%0\n" \ + "0: " insn " %[_val],%[_from]\n" \ + "1: xr %[rc],%[rc]\n" \ "2:\n" \ EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0) \ - : "=d" (__rc), "+d" (val) \ - : "Q" (*(from)) \ + : [rc] "=d" (__rc), [_val] "+d" (val) \ + : [_from] "Q" (*(from)) \ : "cc"); \ __rc; \ }) -- cgit v1.2.3 From 454ede3f1424c36bc124f8d7ed4b3ea654654be1 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sun, 1 May 2022 21:26:06 +0200 Subject: s390/uaccess: use exception handler to zero result on get_user() failure Historically the uaccess code pre-initializes the result of get_user() (and now also __get_kernel_nofault()) to zero and uses the result as input parameter for inline assemblies. This is different to what most, if not all, other architectures are doing, which set the result to zero within the exception handler in case of a fault. Use the new extable mechanism and handle zeroing of the result within the exception handler in case of a fault. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/asm-extable.h | 91 +++++++++++++++++++++++++------------ arch/s390/include/asm/uaccess.h | 72 ++++++++++++++++++----------- arch/s390/mm/extable.c | 39 ++++++++++++++-- 3 files changed, 143 insertions(+), 59 deletions(-) (limited to 'arch/s390') diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index f24d9591aaed..b74f1070ddb2 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -3,12 +3,24 @@ #define __ASM_EXTABLE_H #include +#include #include -#define EX_TYPE_NONE 0 -#define EX_TYPE_FIXUP 1 -#define EX_TYPE_BPF 2 -#define EX_TYPE_UACCESS 3 +#define EX_TYPE_NONE 0 +#define EX_TYPE_FIXUP 1 +#define EX_TYPE_BPF 2 +#define EX_TYPE_UA_STORE 3 +#define EX_TYPE_UA_LOAD_MEM 4 +#define EX_TYPE_UA_LOAD_REG 5 + +#define EX_DATA_REG_ERR_SHIFT 0 +#define EX_DATA_REG_ERR GENMASK(3, 0) + +#define EX_DATA_REG_ADDR_SHIFT 4 +#define EX_DATA_REG_ADDR GENMASK(7, 4) + +#define EX_DATA_LEN_SHIFT 8 +#define EX_DATA_LEN GENMASK(11, 8) #define __EX_TABLE(_section, _fault, _target, _type) \ stringify_in_c(.section _section,"a";) \ @@ -19,35 +31,58 @@ stringify_in_c(.short 0;) \ stringify_in_c(.previous) -#define __EX_TABLE_UA(_section, _fault, _target, _type, _reg) \ - stringify_in_c(.section _section,"a";) \ - stringify_in_c(.align 4;) \ - stringify_in_c(.long (_fault) - .;) \ - stringify_in_c(.long (_target) - .;) \ - stringify_in_c(.short (_type);) \ - stringify_in_c(.macro extable_reg reg;) \ - stringify_in_c(.set .Lfound, 0;) \ - stringify_in_c(.set .Lregnr, 0;) \ - stringify_in_c(.irp rs,r0,r1,r2,r3,r4,r5,r6,r7,r8,r9,r10,r11,r12,r13,r14,r15;) \ - stringify_in_c(.ifc "\reg", "%%\rs";) \ - stringify_in_c(.set .Lfound, 1;) \ - stringify_in_c(.short .Lregnr;) \ - stringify_in_c(.endif;) \ - stringify_in_c(.set .Lregnr, .Lregnr+1;) \ - stringify_in_c(.endr;) \ - stringify_in_c(.ifne (.Lfound != 1);) \ - stringify_in_c(.error "extable_reg: bad register argument";) \ - stringify_in_c(.endif;) \ - stringify_in_c(.endm;) \ - stringify_in_c(extable_reg _reg;) \ - stringify_in_c(.purgem extable_reg;) \ +#define __EX_TABLE_UA(_section, _fault, _target, _type, _regerr, _regaddr, _len)\ + stringify_in_c(.section _section,"a";) \ + stringify_in_c(.align 4;) \ + stringify_in_c(.long (_fault) - .;) \ + stringify_in_c(.long (_target) - .;) \ + stringify_in_c(.short (_type);) \ + stringify_in_c(.macro extable_reg regerr, regaddr;) \ + stringify_in_c(.set .Lfound, 0;) \ + stringify_in_c(.set .Lcurr, 0;) \ + stringify_in_c(.irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15;) \ + stringify_in_c( .ifc "\regerr", "%%r\rs";) \ + stringify_in_c( .set .Lfound, 1;) \ + stringify_in_c( .set .Lregerr, .Lcurr;) \ + stringify_in_c( .endif;) \ + stringify_in_c( .set .Lcurr, .Lcurr+1;) \ + stringify_in_c(.endr;) \ + stringify_in_c(.ifne (.Lfound != 1);) \ + stringify_in_c( .error "extable_reg: bad register argument1";) \ + stringify_in_c(.endif;) \ + stringify_in_c(.set .Lfound, 0;) \ + stringify_in_c(.set .Lcurr, 0;) \ + stringify_in_c(.irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15;) \ + stringify_in_c( .ifc "\regaddr", "%%r\rs";) \ + stringify_in_c( .set .Lfound, 1;) \ + stringify_in_c( .set .Lregaddr, .Lcurr;) \ + stringify_in_c( .endif;) \ + stringify_in_c( .set .Lcurr, .Lcurr+1;) \ + stringify_in_c(.endr;) \ + stringify_in_c(.ifne (.Lfound != 1);) \ + stringify_in_c( .error "extable_reg: bad register argument2";) \ + stringify_in_c(.endif;) \ + stringify_in_c(.short .Lregerr << EX_DATA_REG_ERR_SHIFT | \ + .Lregaddr << EX_DATA_REG_ADDR_SHIFT | \ + _len << EX_DATA_LEN_SHIFT;) \ + stringify_in_c(.endm;) \ + stringify_in_c(extable_reg _regerr,_regaddr;) \ + stringify_in_c(.purgem extable_reg;) \ stringify_in_c(.previous) #define EX_TABLE(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP) + #define EX_TABLE_AMODE31(_fault, _target) \ __EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP) -#define EX_TABLE_UA(_fault, _target, _reg) \ - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UACCESS, _reg) + +#define EX_TABLE_UA_STORE(_fault, _target, _regerr) \ + __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_STORE, _regerr, _regerr, 0) + +#define EX_TABLE_UA_LOAD_MEM(_fault, _target, _regerr, _regmem, _len) \ + __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_MEM, _regerr, _regmem, _len) + +#define EX_TABLE_UA_LOAD_REG(_fault, _target, _regerr, _regzero) \ + __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REG, _regerr, _regzero, 0) #endif /* __ASM_EXTABLE_H */ diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 5ffb88b3fdee..2d8d9e89d48f 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -80,8 +80,12 @@ union oac { }; }; -#define __put_get_user_asm(to, from, size, oac_spec) \ +#define __put_user_asm(to, from, size) \ ({ \ + union oac __oac_spec = { \ + .oac1.as = PSW_BITS_AS_SECONDARY, \ + .oac1.a = 1, \ + }; \ int __rc; \ \ asm volatile( \ @@ -89,26 +93,15 @@ union oac { "0: mvcos %[_to],%[_from],%[_size]\n" \ "1: xr %[rc],%[rc]\n" \ "2:\n" \ - EX_TABLE_UA(0b,2b,%[rc]) EX_TABLE_UA(1b,2b,%[rc]) \ + EX_TABLE_UA_STORE(0b, 2b, %[rc]) \ + EX_TABLE_UA_STORE(1b, 2b, %[rc]) \ : [rc] "=&d" (__rc), [_to] "+Q" (*(to)) \ : [_size] "d" (size), [_from] "Q" (*(from)), \ - [spec] "d" (oac_spec.val) \ + [spec] "d" (__oac_spec.val) \ : "cc", "0"); \ __rc; \ }) -#define __put_user_asm(to, from, size) \ - __put_get_user_asm(to, from, size, ((union oac) { \ - .oac1.as = PSW_BITS_AS_SECONDARY, \ - .oac1.a = 1 \ - })) - -#define __get_user_asm(to, from, size) \ - __put_get_user_asm(to, from, size, ((union oac) { \ - .oac2.as = PSW_BITS_AS_SECONDARY, \ - .oac2.a = 1 \ - })) \ - static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned long size) { int rc; @@ -141,6 +134,29 @@ static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned lon return rc; } +#define __get_user_asm(to, from, size) \ +({ \ + union oac __oac_spec = { \ + .oac2.as = PSW_BITS_AS_SECONDARY, \ + .oac2.a = 1, \ + }; \ + int __rc; \ + \ + asm volatile( \ + " lr 0,%[spec]\n" \ + "0: mvcos 0(%[_to]),%[_from],%[_size]\n" \ + "1: xr %[rc],%[rc]\n" \ + "2:\n" \ + EX_TABLE_UA_LOAD_MEM(0b, 2b, %[rc], %[_to], %[_ksize]) \ + EX_TABLE_UA_LOAD_MEM(1b, 2b, %[rc], %[_to], %[_ksize]) \ + : [rc] "=&d" (__rc), "=Q" (*(to)) \ + : [_size] "d" (size), [_from] "Q" (*(from)), \ + [spec] "d" (__oac_spec.val), [_to] "a" (to), \ + [_ksize] "K" (size) \ + : "cc", "0"); \ + __rc; \ +}) + static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size) { int rc; @@ -210,28 +226,28 @@ static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsign __chk_user_ptr(ptr); \ switch (sizeof(*(ptr))) { \ case 1: { \ - unsigned char __x = 0; \ + unsigned char __x; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 2: { \ - unsigned short __x = 0; \ + unsigned short __x; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 4: { \ - unsigned int __x = 0; \ + unsigned int __x; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 8: { \ - unsigned long long __x = 0; \ + unsigned long __x; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ @@ -281,7 +297,8 @@ int __noreturn __put_kernel_bad(void); "0: " insn " %[_val],%[_to]\n" \ "1: xr %[rc],%[rc]\n" \ "2:\n" \ - EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0) \ + EX_TABLE_UA_STORE(0b, 2b, %[rc]) \ + EX_TABLE_UA_STORE(1b, 2b, %[rc]) \ : [rc] "=d" (__rc), [_to] "+Q" (*(to)) \ : [_val] "d" (val) \ : "cc"); \ @@ -290,7 +307,7 @@ int __noreturn __put_kernel_bad(void); #define __put_kernel_nofault(dst, src, type, err_label) \ do { \ - u64 __x = (u64)(*((type *)(src))); \ + unsigned long __x = (unsigned long)(*((type *)(src))); \ int __pk_err; \ \ switch (sizeof(type)) { \ @@ -324,8 +341,9 @@ int __noreturn __get_kernel_bad(void); "0: " insn " %[_val],%[_from]\n" \ "1: xr %[rc],%[rc]\n" \ "2:\n" \ - EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0) \ - : [rc] "=d" (__rc), [_val] "+d" (val) \ + EX_TABLE_UA_LOAD_REG(0b, 2b, %[rc], %[_val]) \ + EX_TABLE_UA_LOAD_REG(1b, 2b, %[rc], %[_val]) \ + : [rc] "=d" (__rc), [_val] "=d" (val) \ : [_from] "Q" (*(from)) \ : "cc"); \ __rc; \ @@ -337,28 +355,28 @@ do { \ \ switch (sizeof(type)) { \ case 1: { \ - u8 __x = 0; \ + unsigned char __x; \ \ __gk_err = __get_kernel_asm(__x, (type *)(src), "ic"); \ *((type *)(dst)) = (type)__x; \ break; \ }; \ case 2: { \ - u16 __x = 0; \ + unsigned short __x; \ \ __gk_err = __get_kernel_asm(__x, (type *)(src), "lh"); \ *((type *)(dst)) = (type)__x; \ break; \ }; \ case 4: { \ - u32 __x = 0; \ + unsigned int __x; \ \ __gk_err = __get_kernel_asm(__x, (type *)(src), "l"); \ *((type *)(dst)) = (type)__x; \ break; \ }; \ case 8: { \ - u64 __x = 0; \ + unsigned long __x; \ \ __gk_err = __get_kernel_asm(__x, (type *)(src), "lg"); \ *((type *)(dst)) = (type)__x; \ diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index 8ac8ad2474a0..1e4d2187541a 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include +#include #include #include #include @@ -24,9 +26,34 @@ static bool ex_handler_fixup(const struct exception_table_entry *ex, struct pt_r return true; } -static bool ex_handler_uaccess(const struct exception_table_entry *ex, struct pt_regs *regs) +static bool ex_handler_ua_store(const struct exception_table_entry *ex, struct pt_regs *regs) { - regs->gprs[ex->data] = -EFAULT; + unsigned int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); + + regs->gprs[reg_err] = -EFAULT; + regs->psw.addr = extable_fixup(ex); + return true; +} + +static bool ex_handler_ua_load_mem(const struct exception_table_entry *ex, struct pt_regs *regs) +{ + unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data); + unsigned int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); + size_t len = FIELD_GET(EX_DATA_LEN, ex->data); + + regs->gprs[reg_err] = -EFAULT; + memset((void *)regs->gprs[reg_addr], 0, len); + regs->psw.addr = extable_fixup(ex); + return true; +} + +static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex, struct pt_regs *regs) +{ + unsigned int reg_zero = FIELD_GET(EX_DATA_REG_ADDR, ex->data); + unsigned int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); + + regs->gprs[reg_err] = -EFAULT; + regs->gprs[reg_zero] = 0; regs->psw.addr = extable_fixup(ex); return true; } @@ -43,8 +70,12 @@ bool fixup_exception(struct pt_regs *regs) return ex_handler_fixup(ex, regs); case EX_TYPE_BPF: return ex_handler_bpf(ex, regs); - case EX_TYPE_UACCESS: - return ex_handler_uaccess(ex, regs); + case EX_TYPE_UA_STORE: + return ex_handler_ua_store(ex, regs); + case EX_TYPE_UA_LOAD_MEM: + return ex_handler_ua_load_mem(ex, regs); + case EX_TYPE_UA_LOAD_REG: + return ex_handler_ua_load_reg(ex, regs); } panic("invalid exception table entry"); } -- cgit v1.2.3 From a0e3a44bc2a01153fe4ff56fc1ec26832a869437 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sun, 29 May 2022 21:27:48 +0200 Subject: s390/uaccess: use __noreturn instead of __attribute__((noreturn)) Signed-off-by: Heiko Carstens --- arch/s390/include/asm/uaccess.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'arch/s390') diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 2d8d9e89d48f..40361072e638 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -55,9 +55,6 @@ copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned lo return n; } -int __put_user_bad(void) __attribute__((noreturn)); -int __get_user_bad(void) __attribute__((noreturn)); - union oac { unsigned int val; struct { @@ -80,6 +77,8 @@ union oac { }; }; +int __noreturn __put_user_bad(void); + #define __put_user_asm(to, from, size) \ ({ \ union oac __oac_spec = { \ @@ -134,6 +133,8 @@ static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned lon return rc; } +int __noreturn __get_user_bad(void); + #define __get_user_asm(to, from, size) \ ({ \ union oac __oac_spec = { \ -- cgit v1.2.3 From d144182ed19af0bed4e4a73b51c7af9caccf6fa2 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sun, 29 May 2022 21:36:16 +0200 Subject: s390/uaccess: whitespace cleanup Whitespace cleanup to get rid if some checkpatch findings, but mainly to have consistent coding style within the header file again. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/uaccess.h | 132 ++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 66 deletions(-) (limited to 'arch/s390') diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 40361072e638..f4511e21d646 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -3,7 +3,7 @@ * S390 version * Copyright IBM Corp. 1999, 2000 * Author(s): Hartmut Penner (hp@de.ibm.com), - * Martin Schwidefsky (schwidefsky@de.ibm.com) + * Martin Schwidefsky (schwidefsky@de.ibm.com) * * Derived from "include/asm-i386/uaccess.h" */ @@ -194,77 +194,77 @@ static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsign * These are the main single-value transfer routines. They automatically * use the right size if we just have the right pointer type. */ -#define __put_user(x, ptr) \ -({ \ - __typeof__(*(ptr)) __x = (x); \ - int __pu_err = -EFAULT; \ - __chk_user_ptr(ptr); \ - switch (sizeof (*(ptr))) { \ - case 1: \ - case 2: \ - case 4: \ - case 8: \ - __pu_err = __put_user_fn(&__x, ptr, \ - sizeof(*(ptr))); \ - break; \ - default: \ - __put_user_bad(); \ - break; \ - } \ - __builtin_expect(__pu_err, 0); \ +#define __put_user(x, ptr) \ +({ \ + __typeof__(*(ptr)) __x = (x); \ + int __pu_err = -EFAULT; \ + \ + __chk_user_ptr(ptr); \ + switch (sizeof(*(ptr))) { \ + case 1: \ + case 2: \ + case 4: \ + case 8: \ + __pu_err = __put_user_fn(&__x, ptr, sizeof(*(ptr))); \ + break; \ + default: \ + __put_user_bad(); \ + break; \ + } \ + __builtin_expect(__pu_err, 0); \ }) -#define put_user(x, ptr) \ -({ \ - might_fault(); \ - __put_user(x, ptr); \ +#define put_user(x, ptr) \ +({ \ + might_fault(); \ + __put_user(x, ptr); \ }) - -#define __get_user(x, ptr) \ -({ \ - int __gu_err = -EFAULT; \ - __chk_user_ptr(ptr); \ - switch (sizeof(*(ptr))) { \ - case 1: { \ - unsigned char __x; \ - __gu_err = __get_user_fn(&__x, ptr, \ - sizeof(*(ptr))); \ - (x) = *(__force __typeof__(*(ptr)) *) &__x; \ - break; \ - }; \ - case 2: { \ - unsigned short __x; \ - __gu_err = __get_user_fn(&__x, ptr, \ - sizeof(*(ptr))); \ - (x) = *(__force __typeof__(*(ptr)) *) &__x; \ - break; \ - }; \ - case 4: { \ - unsigned int __x; \ - __gu_err = __get_user_fn(&__x, ptr, \ - sizeof(*(ptr))); \ - (x) = *(__force __typeof__(*(ptr)) *) &__x; \ - break; \ - }; \ - case 8: { \ - unsigned long __x; \ - __gu_err = __get_user_fn(&__x, ptr, \ - sizeof(*(ptr))); \ - (x) = *(__force __typeof__(*(ptr)) *) &__x; \ - break; \ - }; \ - default: \ - __get_user_bad(); \ - break; \ - } \ - __builtin_expect(__gu_err, 0); \ +#define __get_user(x, ptr) \ +({ \ + int __gu_err = -EFAULT; \ + \ + __chk_user_ptr(ptr); \ + switch (sizeof(*(ptr))) { \ + case 1: { \ + unsigned char __x; \ + \ + __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ + (x) = *(__force __typeof__(*(ptr)) *)&__x; \ + break; \ + }; \ + case 2: { \ + unsigned short __x; \ + \ + __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ + (x) = *(__force __typeof__(*(ptr)) *)&__x; \ + break; \ + }; \ + case 4: { \ + unsigned int __x; \ + \ + __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ + (x) = *(__force __typeof__(*(ptr)) *)&__x; \ + break; \ + }; \ + case 8: { \ + unsigned long __x; \ + \ + __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr))); \ + (x) = *(__force __typeof__(*(ptr)) *)&__x; \ + break; \ + }; \ + default: \ + __get_user_bad(); \ + break; \ + } \ + __builtin_expect(__gu_err, 0); \ }) -#define get_user(x, ptr) \ -({ \ - might_fault(); \ - __get_user(x, ptr); \ +#define get_user(x, ptr) \ +({ \ + might_fault(); \ + __get_user(x, ptr); \ }) /* -- cgit v1.2.3 From f037acb41dc9fc0f00521685b3250226d6f9b437 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 30 May 2022 14:09:24 +0200 Subject: s390/stack: merge empty stack frame slots Merge empty1 and empty2 arrays within the stack frame to one single array. This is possible since with commit 42b01a553a56 ("s390: always use the packed stack layout") the alternative stack frame layout is gone. Reviewed-by: Nico Boehr Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/include/asm/stacktrace.h | 3 +-- arch/s390/kernel/asm-offsets.c | 10 +++++----- arch/s390/kernel/perf_event.c | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) (limited to 'arch/s390') diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h index f8500191993d..7e0e15bee18d 100644 --- a/arch/s390/include/asm/stacktrace.h +++ b/arch/s390/include/asm/stacktrace.h @@ -39,8 +39,7 @@ static inline bool on_stack(struct stack_info *info, * Kernel uses the packed stack layout (-mpacked-stack). */ struct stack_frame { - unsigned long empty1[5]; - unsigned int empty2[8]; + unsigned long empty[9]; unsigned long gprs[10]; unsigned long back_chain; }; diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index 1e37cad44561..0e10d7ff4203 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -57,11 +57,11 @@ int main(void) /* stack_frame offsets */ OFFSET(__SF_BACKCHAIN, stack_frame, back_chain); OFFSET(__SF_GPRS, stack_frame, gprs); - OFFSET(__SF_EMPTY, stack_frame, empty1[0]); - OFFSET(__SF_SIE_CONTROL, stack_frame, empty1[1]); - OFFSET(__SF_SIE_SAVEAREA, stack_frame, empty1[2]); - OFFSET(__SF_SIE_REASON, stack_frame, empty1[3]); - OFFSET(__SF_SIE_FLAGS, stack_frame, empty1[4]); + OFFSET(__SF_EMPTY, stack_frame, empty[0]); + OFFSET(__SF_SIE_CONTROL, stack_frame, empty[1]); + OFFSET(__SF_SIE_SAVEAREA, stack_frame, empty[2]); + OFFSET(__SF_SIE_REASON, stack_frame, empty[3]); + OFFSET(__SF_SIE_FLAGS, stack_frame, empty[4]); DEFINE(STACK_FRAME_OVERHEAD, sizeof(struct stack_frame)); BLANK(); /* idle data offsets */ diff --git a/arch/s390/kernel/perf_event.c b/arch/s390/kernel/perf_event.c index a7f8db73984b..e259ff1f5ad3 100644 --- a/arch/s390/kernel/perf_event.c +++ b/arch/s390/kernel/perf_event.c @@ -30,7 +30,7 @@ static struct kvm_s390_sie_block *sie_block(struct pt_regs *regs) if (!stack) return NULL; - return (struct kvm_s390_sie_block *)stack->empty1[1]; + return (struct kvm_s390_sie_block *)stack->empty[1]; } static bool is_in_guest(struct pt_regs *regs) -- cgit v1.2.3 From e0ffcf3fe18e0310221461c08969edec2cc7628c Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 30 May 2022 14:20:50 +0200 Subject: s390/stack: add union to reflect kvm stack slot usages Add a union which describes how the empty stack slots are being used by kvm and perf. This should help to avoid another bug like the one which was fixed with commit c9bfb460c3e4 ("s390/perf: obtain sie_block from the right address"). Reviewed-by: Nico Boehr Tested-by: Nico Boehr Signed-off-by: Heiko Carstens --- arch/s390/include/asm/stacktrace.h | 10 +++++++++- arch/s390/kernel/asm-offsets.c | 8 ++++---- arch/s390/kernel/perf_event.c | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) (limited to 'arch/s390') diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h index 7e0e15bee18d..b23c658dce77 100644 --- a/arch/s390/include/asm/stacktrace.h +++ b/arch/s390/include/asm/stacktrace.h @@ -39,7 +39,15 @@ static inline bool on_stack(struct stack_info *info, * Kernel uses the packed stack layout (-mpacked-stack). */ struct stack_frame { - unsigned long empty[9]; + union { + unsigned long empty[9]; + struct { + unsigned long sie_control_block; + unsigned long sie_savearea; + unsigned long sie_reason; + unsigned long sie_flags; + }; + }; unsigned long gprs[10]; unsigned long back_chain; }; diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index 0e10d7ff4203..d8ce965c0a97 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -58,10 +58,10 @@ int main(void) OFFSET(__SF_BACKCHAIN, stack_frame, back_chain); OFFSET(__SF_GPRS, stack_frame, gprs); OFFSET(__SF_EMPTY, stack_frame, empty[0]); - OFFSET(__SF_SIE_CONTROL, stack_frame, empty[1]); - OFFSET(__SF_SIE_SAVEAREA, stack_frame, empty[2]); - OFFSET(__SF_SIE_REASON, stack_frame, empty[3]); - OFFSET(__SF_SIE_FLAGS, stack_frame, empty[4]); + OFFSET(__SF_SIE_CONTROL, stack_frame, sie_control_block); + OFFSET(__SF_SIE_SAVEAREA, stack_frame, sie_savearea); + OFFSET(__SF_SIE_REASON, stack_frame, sie_reason); + OFFSET(__SF_SIE_FLAGS, stack_frame, sie_flags); DEFINE(STACK_FRAME_OVERHEAD, sizeof(struct stack_frame)); BLANK(); /* idle data offsets */ diff --git a/arch/s390/kernel/perf_event.c b/arch/s390/kernel/perf_event.c index e259ff1f5ad3..c27321cb0969 100644 --- a/arch/s390/kernel/perf_event.c +++ b/arch/s390/kernel/perf_event.c @@ -30,7 +30,7 @@ static struct kvm_s390_sie_block *sie_block(struct pt_regs *regs) if (!stack) return NULL; - return (struct kvm_s390_sie_block *)stack->empty[1]; + return (struct kvm_s390_sie_block *)stack->sie_control_block; } static bool is_in_guest(struct pt_regs *regs) -- cgit v1.2.3