diff options
| -rw-r--r-- | arch/s390/kernel/entry.S | 75 | ||||
| -rw-r--r-- | arch/s390/kernel/entry64.S | 108 |
2 files changed, 64 insertions, 119 deletions
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 6af7045280a8..ffebfb64b913 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -301,31 +301,29 @@ sysc_restore_trace_psw: #endif # -# recheck if there is more work to do -# -sysc_work_loop: - tm __TI_flags+3(%r9),_TIF_WORK_SVC - bz BASED(sysc_restore) # there is no work to do -# -# One of the work bits is on. Find out which one. +# There is work to do, but first we need to check if we return to userspace. # sysc_work: tm SP_PSW+1(%r15),0x01 # returning to user ? bno BASED(sysc_restore) + +# +# One of the work bits is on. Find out which one. +# +sysc_work_loop: tm __TI_flags+3(%r9),_TIF_MCCK_PENDING bo BASED(sysc_mcck_pending) tm __TI_flags+3(%r9),_TIF_NEED_RESCHED bo BASED(sysc_reschedule) tm __TI_flags+3(%r9),_TIF_SIGPENDING - bnz BASED(sysc_sigpending) + bo BASED(sysc_sigpending) tm __TI_flags+3(%r9),_TIF_NOTIFY_RESUME - bnz BASED(sysc_notify_resume) + bo BASED(sysc_notify_resume) tm __TI_flags+3(%r9),_TIF_RESTART_SVC bo BASED(sysc_restart) tm __TI_flags+3(%r9),_TIF_SINGLE_STEP bo BASED(sysc_singlestep) - b BASED(sysc_restore) -sysc_work_done: + b BASED(sysc_return) # beware of critical section cleanup # # _TIF_NEED_RESCHED is set, call schedule @@ -386,7 +384,7 @@ sysc_singlestep: mvi SP_SVCNR+1(%r15),0xff la %r2,SP_PTREGS(%r15) # address of register-save area l %r1,BASED(.Lhandle_per) # load adr. of per handler - la %r14,BASED(sysc_return) # load adr. of system return + la %r14,BASED(sysc_work_loop) # load adr. of system return br %r1 # branch to do_single_step # @@ -636,30 +634,36 @@ io_restore_trace_psw: #endif # -# switch to kernel stack, then check the TIF bits +# There is work todo, find out in which context we have been interrupted: +# 1) if we return to user space we can do all _TIF_WORK_INT work +# 2) if we return to kernel code and preemptive scheduling is enabled check +# the preemption counter and if it is zero call preempt_schedule_irq +# Before any work can be done, a switch to the kernel stack is required. # io_work: tm SP_PSW+1(%r15),0x01 # returning to user ? -#ifndef CONFIG_PREEMPT - bno BASED(io_restore) # no-> skip resched & signal -#else - bnz BASED(io_work_user) # no -> check for preemptive scheduling + bo BASED(io_work_user) # yes -> do resched & signal +#ifdef CONFIG_PREEMPT # check for preemptive scheduling icm %r0,15,__TI_precount(%r9) bnz BASED(io_restore) # preemption disabled + # switch to kernel stack l %r1,SP_R15(%r15) s %r1,BASED(.Lc_spsize) mvc SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15) xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) # clear back chain lr %r15,%r1 io_resume_loop: - tm __TI_flags+3(%r9),_TIF_NEED_RESCHED - bno BASED(io_restore) l %r1,BASED(.Lpreempt_schedule_irq) la %r14,BASED(io_resume_loop) - br %r1 # call schedule + tm __TI_flags+3(%r9),_TIF_NEED_RESCHED + bor %r1 # call preempt_schedule_irq #endif + b BASED(io_restore) +# +# Need to do work before returning to userspace, switch to kernel stack +# io_work_user: l %r1,__LC_KERNEL_STACK s %r1,BASED(.Lc_spsize) @@ -668,7 +672,7 @@ io_work_user: lr %r15,%r1 # # One of the work bits is on. Find out which one. -# Checked are: _TIF_SIGPENDING, _TIF_NEED_RESCHED +# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED # and _TIF_MCCK_PENDING # io_work_loop: @@ -677,11 +681,10 @@ io_work_loop: tm __TI_flags+3(%r9),_TIF_NEED_RESCHED bo BASED(io_reschedule) tm __TI_flags+3(%r9),_TIF_SIGPENDING - bnz BASED(io_sigpending) + bo BASED(io_sigpending) tm __TI_flags+3(%r9),_TIF_NOTIFY_RESUME - bnz BASED(io_notify_resume) - b BASED(io_restore) -io_work_done: + bo BASED(io_notify_resume) + b BASED(io_return) # beware of critical section cleanup # # _TIF_MCCK_PENDING is set, call handler @@ -701,8 +704,6 @@ io_reschedule: basr %r14,%r1 # call scheduler stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts TRACE_IRQS_OFF - tm __TI_flags+3(%r9),_TIF_WORK_INT - bz BASED(io_restore) # there is no work to do b BASED(io_work_loop) # @@ -921,14 +922,10 @@ cleanup_table_sysc_return: .long sysc_return + 0x80000000, sysc_leave + 0x80000000 cleanup_table_sysc_leave: .long sysc_leave + 0x80000000, sysc_done + 0x80000000 -cleanup_table_sysc_work_loop: - .long sysc_work_loop + 0x80000000, sysc_work_done + 0x80000000 cleanup_table_io_return: .long io_return + 0x80000000, io_leave + 0x80000000 cleanup_table_io_leave: .long io_leave + 0x80000000, io_done + 0x80000000 -cleanup_table_io_work_loop: - .long io_work_loop + 0x80000000, io_work_done + 0x80000000 cleanup_critical: clc 4(4,%r12),BASED(cleanup_table_system_call) @@ -946,11 +943,6 @@ cleanup_critical: clc 4(4,%r12),BASED(cleanup_table_sysc_leave+4) bl BASED(cleanup_sysc_leave) 0: - clc 4(4,%r12),BASED(cleanup_table_sysc_work_loop) - bl BASED(0f) - clc 4(4,%r12),BASED(cleanup_table_sysc_work_loop+4) - bl BASED(cleanup_sysc_return) -0: clc 4(4,%r12),BASED(cleanup_table_io_return) bl BASED(0f) clc 4(4,%r12),BASED(cleanup_table_io_return+4) @@ -961,11 +953,6 @@ cleanup_critical: clc 4(4,%r12),BASED(cleanup_table_io_leave+4) bl BASED(cleanup_io_leave) 0: - clc 4(4,%r12),BASED(cleanup_table_io_work_loop) - bl BASED(0f) - clc 4(4,%r12),BASED(cleanup_table_io_work_loop+4) - bl BASED(cleanup_io_work_loop) -0: br %r14 cleanup_system_call: @@ -1043,12 +1030,6 @@ cleanup_io_return: la %r12,__LC_RETURN_PSW br %r14 -cleanup_io_work_loop: - mvc __LC_RETURN_PSW(4),0(%r12) - mvc __LC_RETURN_PSW+4(4),BASED(cleanup_table_io_work_loop) - la %r12,__LC_RETURN_PSW - br %r14 - cleanup_io_leave: clc 4(4,%r12),BASED(cleanup_io_leave_insn) be BASED(2f) diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S index 52106d53271c..ca02b10a2c32 100644 --- a/arch/s390/kernel/entry64.S +++ b/arch/s390/kernel/entry64.S @@ -291,38 +291,36 @@ sysc_restore_trace_psw: #endif # -# recheck if there is more work to do -# -sysc_work_loop: - tm __TI_flags+7(%r9),_TIF_WORK_SVC - jz sysc_restore # there is no work to do -# -# One of the work bits is on. Find out which one. +# There is work to do, but first we need to check if we return to userspace. # sysc_work: tm SP_PSW+1(%r15),0x01 # returning to user ? jno sysc_restore + +# +# One of the work bits is on. Find out which one. +# +sysc_work_loop: tm __TI_flags+7(%r9),_TIF_MCCK_PENDING jo sysc_mcck_pending tm __TI_flags+7(%r9),_TIF_NEED_RESCHED jo sysc_reschedule tm __TI_flags+7(%r9),_TIF_SIGPENDING - jnz sysc_sigpending + jo sysc_sigpending tm __TI_flags+7(%r9),_TIF_NOTIFY_RESUME - jnz sysc_notify_resume + jo sysc_notify_resume tm __TI_flags+7(%r9),_TIF_RESTART_SVC jo sysc_restart tm __TI_flags+7(%r9),_TIF_SINGLE_STEP jo sysc_singlestep - j sysc_restore -sysc_work_done: + j sysc_return # beware of critical section cleanup # # _TIF_NEED_RESCHED is set, call schedule # sysc_reschedule: larl %r14,sysc_work_loop - jg schedule # return point is sysc_return + jg schedule # return point is sysc_work_loop # # _TIF_MCCK_PENDING is set, call handler @@ -369,7 +367,7 @@ sysc_singlestep: ni __TI_flags+7(%r9),255-_TIF_SINGLE_STEP # clear TIF_SINGLE_STEP xc SP_SVCNR(2,%r15),SP_SVCNR(%r15) # clear svc number la %r2,SP_PTREGS(%r15) # address of register-save area - larl %r14,sysc_return # load adr. of system return + larl %r14,sysc_work_loop # load adr. of system return jg do_single_step # branch to do_sigtrap # @@ -605,37 +603,27 @@ io_restore_trace_psw: #endif # -# There is work todo, we need to check if we return to userspace, then -# check, if we are in SIE, if yes leave it +# There is work todo, find out in which context we have been interrupted: +# 1) if we return to user space we can do all _TIF_WORK_INT work +# 2) if we return to kernel code and kvm is enabled check if we need to +# modify the psw to leave SIE +# 3) if we return to kernel code and preemptive scheduling is enabled check +# the preemption counter and if it is zero call preempt_schedule_irq +# Before any work can be done, a switch to the kernel stack is required. # io_work: tm SP_PSW+1(%r15),0x01 # returning to user ? -#ifndef CONFIG_PREEMPT -#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) - jnz io_work_user # yes -> no need to check for SIE - la %r1, BASED(sie_opcode) # we return to kernel here - lg %r2, SP_PSW+8(%r15) - clc 0(2,%r1), 0(%r2) # is current instruction = SIE? - jne io_restore # no-> return to kernel - lg %r1, SP_PSW+8(%r15) # yes-> add 4 bytes to leave SIE - aghi %r1, 4 - stg %r1, SP_PSW+8(%r15) - j io_restore # return to kernel -#else - jno io_restore # no-> skip resched & signal -#endif -#else - jnz io_work_user # yes -> do resched & signal + jo io_work_user # yes -> do resched & signal #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) - la %r1, BASED(sie_opcode) - lg %r2, SP_PSW+8(%r15) - clc 0(2,%r1), 0(%r2) # is current instruction = SIE? + lg %r2,SP_PSW+8(%r15) # check if current instruction is SIE + lh %r1,0(%r2) + chi %r1,-19948 # signed 16 bit compare with 0xb214 jne 0f # no -> leave PSW alone - lg %r1, SP_PSW+8(%r15) # yes-> add 4 bytes to leave SIE - aghi %r1, 4 - stg %r1, SP_PSW+8(%r15) + aghi %r2,4 # yes-> add 4 bytes to leave SIE + stg %r2,SP_PSW+8(%r15) 0: #endif +#ifdef CONFIG_PREEMPT # check for preemptive scheduling icm %r0,15,__TI_precount(%r9) jnz io_restore # preemption is disabled @@ -646,21 +634,25 @@ io_work: xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) # clear back chain lgr %r15,%r1 io_resume_loop: - tm __TI_flags+7(%r9),_TIF_NEED_RESCHED - jno io_restore larl %r14,io_resume_loop - jg preempt_schedule_irq + tm __TI_flags+7(%r12),_TIF_NEED_RESCHED + jgo preempt_schedule_irq #endif + j io_restore +# +# Need to do work before returning to userspace, switch to kernel stack +# io_work_user: lg %r1,__LC_KERNEL_STACK aghi %r1,-SP_SIZE mvc SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15) xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) # clear back chain lgr %r15,%r1 + # # One of the work bits is on. Find out which one. -# Checked are: _TIF_SIGPENDING, _TIF_RESTORE_SIGPENDING, _TIF_NEED_RESCHED +# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED # and _TIF_MCCK_PENDING # io_work_loop: @@ -669,16 +661,10 @@ io_work_loop: tm __TI_flags+7(%r9),_TIF_NEED_RESCHED jo io_reschedule tm __TI_flags+7(%r9),_TIF_SIGPENDING - jnz io_sigpending + jo io_sigpending tm __TI_flags+7(%r9),_TIF_NOTIFY_RESUME - jnz io_notify_resume - j io_restore -io_work_done: - -#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) -sie_opcode: - .long 0xb2140000 -#endif + jo io_notify_resume + j io_return # beware of critical section cleanup # # _TIF_MCCK_PENDING is set, call handler @@ -696,8 +682,6 @@ io_reschedule: brasl %r14,schedule # call scheduler stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts TRACE_IRQS_OFF - tm __TI_flags+7(%r9),_TIF_WORK_INT - jz io_restore # there is no work to do j io_work_loop # @@ -903,14 +887,10 @@ cleanup_table_sysc_return: .quad sysc_return, sysc_leave cleanup_table_sysc_leave: .quad sysc_leave, sysc_done -cleanup_table_sysc_work_loop: - .quad sysc_work_loop, sysc_work_done cleanup_table_io_return: .quad io_return, io_leave cleanup_table_io_leave: .quad io_leave, io_done -cleanup_table_io_work_loop: - .quad io_work_loop, io_work_done cleanup_critical: clc 8(8,%r12),BASED(cleanup_table_system_call) @@ -928,11 +908,6 @@ cleanup_critical: clc 8(8,%r12),BASED(cleanup_table_sysc_leave+8) jl cleanup_sysc_leave 0: - clc 8(8,%r12),BASED(cleanup_table_sysc_work_loop) - jl 0f - clc 8(8,%r12),BASED(cleanup_table_sysc_work_loop+8) - jl cleanup_sysc_return -0: clc 8(8,%r12),BASED(cleanup_table_io_return) jl 0f clc 8(8,%r12),BASED(cleanup_table_io_return+8) @@ -943,11 +918,6 @@ cleanup_critical: clc 8(8,%r12),BASED(cleanup_table_io_leave+8) jl cleanup_io_leave 0: - clc 8(8,%r12),BASED(cleanup_table_io_work_loop) - jl 0f - clc 8(8,%r12),BASED(cleanup_table_io_work_loop+8) - jl cleanup_io_work_loop -0: br %r14 cleanup_system_call: @@ -1025,12 +995,6 @@ cleanup_io_return: la %r12,__LC_RETURN_PSW br %r14 -cleanup_io_work_loop: - mvc __LC_RETURN_PSW(8),0(%r12) - mvc __LC_RETURN_PSW+8(8),BASED(cleanup_table_io_work_loop) - la %r12,__LC_RETURN_PSW - br %r14 - cleanup_io_leave: clc 8(8,%r12),BASED(cleanup_io_leave_insn) je 3f |