From 624db9eabc74597f682c0651047a25b54f7260a1 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:51 +0100 Subject: x86: Select CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK Now that all invocations of irq_exit_rcu() happen on the irq stack, turn on CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK which causes the core code to invoke __do_softirq() directly without going through do_softirq_own_stack(). That means do_softirq_own_stack() is only invoked from task context which means it can't be on the irq stack. Remove the conditional from run_softirq_on_irqstack_cond() and rename the function accordingly. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002513.068033456@linutronix.de --- arch/x86/Kconfig | 1 + arch/x86/include/asm/irq_stack.h | 19 ++++++++----------- arch/x86/kernel/irq_64.c | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 21f851179ff0..e17ce871bb19 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -187,6 +187,7 @@ config X86 select HAVE_HW_BREAKPOINT select HAVE_IDE select HAVE_IOREMAP_PROT + select HAVE_IRQ_EXIT_ON_IRQ_STACK if X86_64 select HAVE_IRQ_TIME_ACCOUNTING select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_GZIP diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index fa444c27772a..1b82f9230709 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -189,19 +189,16 @@ "call %P[__func] \n" /* - * Macro to invoke __do_softirq on the irq stack. Contrary to the above - * the only check which is necessary is whether the interrupt stack is - * in use already. + * Macro to invoke __do_softirq on the irq stack. This is only called from + * task context when bottom halfs are about to be reenabled and soft + * interrupts are pending to be processed. The interrupt stack cannot be in + * use here. */ -#define run_softirq_on_irqstack_cond() \ +#define run_softirq_on_irqstack() \ { \ - if (__this_cpu_read(hardirq_stack_inuse)) { \ - __do_softirq(); \ - } else { \ - __this_cpu_write(hardirq_stack_inuse, true); \ - call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \ - __this_cpu_write(hardirq_stack_inuse, false); \ - } \ + __this_cpu_write(hardirq_stack_inuse, true); \ + call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \ + __this_cpu_write(hardirq_stack_inuse, false); \ } #else /* CONFIG_X86_64 */ diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index 8d9f9a1b49e5..b88fdb9686e6 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -76,5 +76,5 @@ int irq_init_percpu_irqstack(unsigned int cpu) void do_softirq_own_stack(void) { - run_softirq_on_irqstack_cond(); + run_softirq_on_irqstack(); } -- cgit v1.2.3