From 7652ac92018536eb807b6c2130100c85f1ba7e3b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Jul 2019 21:42:46 +0200 Subject: x86/asm: Move native_write_cr0/4() out of line The pinning of sensitive CR0 and CR4 bits caused a boot crash when loading the kvm_intel module on a kernel compiled with CONFIG_PARAVIRT=n. The reason is that the static key which controls the pinning is marked RO after init. The kvm_intel module contains a CR4 write which requires to update the static key entry list. That obviously does not work when the key is in a RO section. With CONFIG_PARAVIRT enabled this does not happen because the CR4 write uses the paravirt indirection and the actual write function is built in. As the key is intended to be immutable after init, move native_write_cr0/4() out of line. While at it consolidate the update of the cr4 shadow variable and store the value right away when the pinning is initialized on a booting CPU. No point in reading it back 20 instructions later. This allows to confine the static key and the pinning variable to cpu/common and allows to mark them static. Fixes: 8dbec27a242c ("x86/asm: Pin sensitive CR0 bits") Fixes: 873d50d58f67 ("x86/asm: Pin sensitive CR4 bits") Reported-by: Linus Torvalds Reported-by: Xi Ruoyao Signed-off-by: Thomas Gleixner Tested-by: Xi Ruoyao Acked-by: Kees Cook Acked-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/alpine.DEB.2.21.1907102140340.1758@nanos.tec.linutronix.de --- arch/x86/include/asm/processor.h | 1 + arch/x86/include/asm/special_insns.h | 41 ++---------------------------------- 2 files changed, 3 insertions(+), 39 deletions(-) (limited to 'arch/x86/include') diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 3eab6ece52b4..6e0a3b43d027 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -741,6 +741,7 @@ extern void load_direct_gdt(int); extern void load_fixmap_gdt(int); extern void load_percpu_segment(int); extern void cpu_init(void); +extern void cr4_init(void); static inline unsigned long get_debugctlmsr(void) { diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h index b2e84d113f2a..219be88a59d2 100644 --- a/arch/x86/include/asm/special_insns.h +++ b/arch/x86/include/asm/special_insns.h @@ -18,9 +18,7 @@ */ extern unsigned long __force_order; -/* Starts false and gets enabled once CPU feature detection is done. */ -DECLARE_STATIC_KEY_FALSE(cr_pinning); -extern unsigned long cr4_pinned_bits; +void native_write_cr0(unsigned long val); static inline unsigned long native_read_cr0(void) { @@ -29,24 +27,6 @@ static inline unsigned long native_read_cr0(void) return val; } -static inline void native_write_cr0(unsigned long val) -{ - unsigned long bits_missing = 0; - -set_register: - asm volatile("mov %0,%%cr0": "+r" (val), "+m" (__force_order)); - - if (static_branch_likely(&cr_pinning)) { - if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) { - bits_missing = X86_CR0_WP; - val |= bits_missing; - goto set_register; - } - /* Warn after we've set the missing bits. */ - WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n"); - } -} - static inline unsigned long native_read_cr2(void) { unsigned long val; @@ -91,24 +71,7 @@ static inline unsigned long native_read_cr4(void) return val; } -static inline void native_write_cr4(unsigned long val) -{ - unsigned long bits_missing = 0; - -set_register: - asm volatile("mov %0,%%cr4": "+r" (val), "+m" (cr4_pinned_bits)); - - if (static_branch_likely(&cr_pinning)) { - if (unlikely((val & cr4_pinned_bits) != cr4_pinned_bits)) { - bits_missing = ~val & cr4_pinned_bits; - val |= bits_missing; - goto set_register; - } - /* Warn after we've set the missing bits. */ - WARN_ONCE(bits_missing, "CR4 bits went missing: %lx!?\n", - bits_missing); - } -} +void native_write_cr4(unsigned long val); #ifdef CONFIG_X86_64 static inline unsigned long native_read_cr8(void) -- cgit v1.2.3