summaryrefslogtreecommitdiffstats
path: root/arch/powerpc/kvm/book3s_64_entry.S
blob: a9ab92abffe8ea6a45696d487cceae1da61222d5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
/* SPDX-License-Identifier: GPL-2.0-only */
#include <linux/export.h>
#include <asm/asm-offsets.h>
#include <asm/cache.h>
#include <asm/code-patching-asm.h>
#include <asm/exception-64s.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_book3s_asm.h>
#include <asm/mmu.h>
#include <asm/ppc_asm.h>
#include <asm/ptrace.h>
#include <asm/reg.h>
#include <asm/ultravisor-api.h>

/*
 * These are branched to from interrupt handlers in exception-64s.S which set
 * IKVM_REAL or IKVM_VIRT, if HSTATE_IN_GUEST was found to be non-zero.
 */

/*
 * This is a hcall, so register convention is as
 * Documentation/arch/powerpc/papr_hcalls.rst.
 *
 * This may also be a syscall from PR-KVM userspace that is to be
 * reflected to the PR guest kernel, so registers may be set up for
 * a system call rather than hcall. We don't currently clobber
 * anything here, but the 0xc00 handler has already clobbered CTR
 * and CR0, so PR-KVM can not support a guest kernel that preserves
 * those registers across its system calls.
 *
 * The state of registers is as kvmppc_interrupt, except CFAR is not
 * saved, R13 is not in SCRATCH0, and R10 does not contain the trap.
 */
.global	kvmppc_hcall
.balign IFETCH_ALIGN_BYTES
kvmppc_hcall:
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
	lbz	r10,HSTATE_IN_GUEST(r13)
	cmpwi	r10,KVM_GUEST_MODE_HV_P9
	beq	kvmppc_p9_exit_hcall
#endif
	ld	r10,PACA_EXGEN+EX_R13(r13)
	SET_SCRATCH0(r10)
	li	r10,0xc00
	/* Now we look like kvmppc_interrupt */
	li	r11,PACA_EXGEN
	b	.Lgot_save_area

/*
 * KVM interrupt entry occurs after GEN_INT_ENTRY runs, and follows that
 * call convention:
 *
 * guest R9-R13, CTR, CFAR, PPR saved in PACA EX_xxx save area
 * guest (H)DAR, (H)DSISR are also in the save area for relevant interrupts
 * guest R13 also saved in SCRATCH0
 * R13		= PACA
 * R11		= (H)SRR0
 * R12		= (H)SRR1
 * R9		= guest CR
 * PPR is set to medium
 *
 * With the addition for KVM:
 * R10		= trap vector
 */
.global	kvmppc_interrupt
.balign IFETCH_ALIGN_BYTES
kvmppc_interrupt:
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
	std	r10,HSTATE_SCRATCH0(r13)
	lbz	r10,HSTATE_IN_GUEST(r13)
	cmpwi	r10,KVM_GUEST_MODE_HV_P9
	beq	kvmppc_p9_exit_interrupt
	ld	r10,HSTATE_SCRATCH0(r13)
#endif
	li	r11,PACA_EXGEN
	cmpdi	r10,0x200
	bgt+	.Lgot_save_area
	li	r11,PACA_EXMC
	beq	.Lgot_save_area
	li	r11,PACA_EXNMI
.Lgot_save_area:
	add	r11,r11,r13
BEGIN_FTR_SECTION
	ld	r12,EX_CFAR(r11)
	std	r12,HSTATE_CFAR(r13)
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
	ld	r12,EX_CTR(r11)
	mtctr	r12
BEGIN_FTR_SECTION
	ld	r12,EX_PPR(r11)
	std	r12,HSTATE_PPR(r13)
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
	ld	r12,EX_R12(r11)
	std	r12,HSTATE_SCRATCH0(r13)
	sldi	r12,r9,32
	or	r12,r12,r10
	ld	r9,EX_R9(r11)
	ld	r10,EX_R10(r11)
	ld	r11,EX_R11(r11)

	/*
	 * Hcalls and other interrupts come here after normalising register
	 * contents and save locations:
	 *
	 * R12		= (guest CR << 32) | interrupt vector
	 * R13		= PACA
	 * guest R12 saved in shadow HSTATE_SCRATCH0
	 * guest R13 saved in SPRN_SCRATCH0
	 */
	std	r9,HSTATE_SCRATCH2(r13)
	lbz	r9,HSTATE_IN_GUEST(r13)
	cmpwi	r9,KVM_GUEST_MODE_SKIP
	beq-	.Lmaybe_skip
.Lno_skip:
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
	cmpwi	r9,KVM_GUEST_MODE_GUEST
	beq	kvmppc_interrupt_pr
#endif
	b	kvmppc_interrupt_hv
#else
	b	kvmppc_interrupt_pr
#endif

/*
 * "Skip" interrupts are part of a trick KVM uses a with hash guests to load
 * the faulting instruction in guest memory from the hypervisor without
 * walking page tables.
 *
 * When the guest takes a fault that requires the hypervisor to load the
 * instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1
 * and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets
 * MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions
 * but loads and stores will access the guest context. This is used to load
 * the faulting instruction using the faulting guest effective address.
 *
 * However the guest context may not be able to translate, or it may cause a
 * machine check or other issue, which results in a fault in the host
 * (even with KVM-HV).
 *
 * These faults come here because KVM_GUEST_MODE_SKIP was set, so if they
 * are (or are likely) caused by that load, the instruction is skipped by
 * just returning with the PC advanced +4, where it is noticed the load did
 * not execute and it goes to the slow path which walks the page tables to
 * read guest memory.
 */
.Lmaybe_skip:
	cmpwi	r12,BOOK3S_INTERRUPT_MACHINE_CHECK
	beq	1f
	cmpwi	r12,BOOK3S_INTERRUPT_DATA_STORAGE
	beq	1f
	cmpwi	r12,BOOK3S_INTERRUPT_DATA_SEGMENT
	beq	1f
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
	/* HSRR interrupts get 2 added to interrupt number */
	cmpwi	r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2
	beq	2f
#endif
	b	.Lno_skip
1:	mfspr	r9,SPRN_SRR0
	addi	r9,r9,4
	mtspr	SPRN_SRR0,r9
	ld	r12,HSTATE_SCRATCH0(r13)
	ld	r9,HSTATE_SCRATCH2(r13)
	GET_SCRATCH0(r13)
	RFI_TO_KERNEL
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
2:	mfspr	r9,SPRN_HSRR0
	addi	r9,r9,4
	mtspr	SPRN_HSRR0,r9
	ld	r12,HSTATE_SCRATCH0(r13)
	ld	r9,HSTATE_SCRATCH2(r13)
	GET_SCRATCH0(r13)
	HRFI_TO_KERNEL
#endif

#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE

/* Stack frame offsets for kvmppc_p9_enter_guest */
#define SFS			(144 + STACK_FRAME_MIN_SIZE)
#define STACK_SLOT_NVGPRS	(SFS - 144)	/* 18 gprs */

/*
 * void kvmppc_p9_enter_guest(struct vcpu *vcpu);
 *
 * Enter the guest on a ISAv3.0 or later system.
 */
.balign	IFETCH_ALIGN_BYTES
_GLOBAL(kvmppc_p9_enter_guest)
EXPORT_SYMBOL_GPL(kvmppc_p9_enter_guest)
	mflr	r0
	std	r0,PPC_LR_STKOFF(r1)
	stdu	r1,-SFS(r1)

	std	r1,HSTATE_HOST_R1(r13)

	mfcr	r4
	stw	r4,SFS+8(r1)

	reg = 14
	.rept	18
	std	reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
	reg = reg + 1
	.endr

	ld	r4,VCPU_LR(r3)
	mtlr	r4
	ld	r4,VCPU_CTR(r3)
	mtctr	r4
	ld	r4,VCPU_XER(r3)
	mtspr	SPRN_XER,r4

	ld	r1,VCPU_CR(r3)

BEGIN_FTR_SECTION
	ld	r4,VCPU_CFAR(r3)
	mtspr	SPRN_CFAR,r4
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
BEGIN_FTR_SECTION
	ld	r4,VCPU_PPR(r3)
	mtspr	SPRN_PPR,r4
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)

	reg = 4
	.rept	28
	ld	reg,__VCPU_GPR(reg)(r3)
	reg = reg + 1
	.endr

	ld	r4,VCPU_KVM(r3)
	lbz	r4,KVM_SECURE_GUEST(r4)
	cmpdi	r4,0
	ld	r4,VCPU_GPR(R4)(r3)
	bne	.Lret_to_ultra

	mtcr	r1

	ld	r0,VCPU_GPR(R0)(r3)
	ld	r1,VCPU_GPR(R1)(r3)
	ld	r2,VCPU_GPR(R2)(r3)
	ld	r3,VCPU_GPR(R3)(r3)

	HRFI_TO_GUEST
	b	.

	/*
	 * Use UV_RETURN ultracall to return control back to the Ultravisor
	 * after processing an hypercall or interrupt that was forwarded
	 * (a.k.a. reflected) to the Hypervisor.
	 *
	 * All registers have already been reloaded except the ucall requires:
	 *   R0 = hcall result
	 *   R2 = SRR1, so UV can detect a synthesized interrupt (if any)
	 *   R3 = UV_RETURN
	 */
.Lret_to_ultra:
	mtcr	r1
	ld	r1,VCPU_GPR(R1)(r3)

	ld	r0,VCPU_GPR(R3)(r3)
	mfspr	r2,SPRN_SRR1
	LOAD_REG_IMMEDIATE(r3, UV_RETURN)
	sc	2

/*
 * kvmppc_p9_exit_hcall and kvmppc_p9_exit_interrupt are branched to from
 * above if the interrupt was taken for a guest that was entered via
 * kvmppc_p9_enter_guest().
 *
 * The exit code recovers the host stack and vcpu pointer, saves all guest GPRs
 * and CR, LR, XER as well as guest MSR and NIA into the VCPU, then re-
 * establishes the host stack and registers to return from the
 * kvmppc_p9_enter_guest() function, which saves CTR and other guest registers
 * (SPRs and FP, VEC, etc).
 */
.balign	IFETCH_ALIGN_BYTES
kvmppc_p9_exit_hcall:
	mfspr	r11,SPRN_SRR0
	mfspr	r12,SPRN_SRR1
	li	r10,0xc00
	std	r10,HSTATE_SCRATCH0(r13)

.balign	IFETCH_ALIGN_BYTES
kvmppc_p9_exit_interrupt:
	/*
	 * If set to KVM_GUEST_MODE_HV_P9 but we're still in the
	 * hypervisor, that means we can't return from the entry stack.
	 */
	rldicl. r10,r12,64-MSR_HV_LG,63
	bne-	kvmppc_p9_bad_interrupt

	std     r1,HSTATE_SCRATCH1(r13)
	std     r3,HSTATE_SCRATCH2(r13)
	ld	r1,HSTATE_HOST_R1(r13)
	ld	r3,HSTATE_KVM_VCPU(r13)

	std	r9,VCPU_CR(r3)

1:
	std	r11,VCPU_PC(r3)
	std	r12,VCPU_MSR(r3)

	reg = 14
	.rept	18
	std	reg,__VCPU_GPR(reg)(r3)
	reg = reg + 1
	.endr

	/* r1, r3, r9-r13 are saved to vcpu by C code */
	std	r0,VCPU_GPR(R0)(r3)
	std	r2,VCPU_GPR(R2)(r3)
	reg = 4
	.rept	5
	std	reg,__VCPU_GPR(reg)(r3)
	reg = reg + 1
	.endr

	LOAD_PACA_TOC()

	mflr	r4
	std	r4,VCPU_LR(r3)
	mfspr	r4,SPRN_XER
	std	r4,VCPU_XER(r3)

	reg = 14
	.rept	18
	ld	reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
	reg = reg + 1
	.endr

	lwz	r4,SFS+8(r1)
	mtcr	r4

	/*
	 * Flush the link stack here, before executing the first blr on the
	 * way out of the guest.
	 *
	 * The link stack won't match coming out of the guest anyway so the
	 * only cost is the flush itself. The call clobbers r0.
	 */
1:	nop
	patch_site 1b patch__call_kvm_flush_link_stack_p9

	addi	r1,r1,SFS
	ld	r0,PPC_LR_STKOFF(r1)
	mtlr	r0
	blr

/*
 * Took an interrupt somewhere right before HRFID to guest, so registers are
 * in a bad way. Return things hopefully enough to run host virtual code and
 * run the Linux interrupt handler (SRESET or MCE) to print something useful.
 *
 * We could be really clever and save all host registers in known locations
 * before setting HSTATE_IN_GUEST, then restoring them all here, and setting
 * return address to a fixup that sets them up again. But that's a lot of
 * effort for a small bit of code. Lots of other things to do first.
 */
kvmppc_p9_bad_interrupt:
BEGIN_MMU_FTR_SECTION
	/*
	 * Hash host doesn't try to recover MMU (requires host SLB reload)
	 */
	b	.
END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
	/*
	 * Clean up guest registers to give host a chance to run.
	 */
	li	r10,0
	mtspr	SPRN_AMR,r10
	mtspr	SPRN_IAMR,r10
	mtspr	SPRN_CIABR,r10
	mtspr	SPRN_DAWRX0,r10
BEGIN_FTR_SECTION
	mtspr	SPRN_DAWRX1,r10
END_FTR_SECTION_IFSET(CPU_FTR_DAWR1)

	/*
	 * Switch to host MMU mode (don't have the real host PID but we aren't
	 * going back to userspace).
	 */
	hwsync
	isync

	mtspr	SPRN_PID,r10

	ld	r10, HSTATE_KVM_VCPU(r13)
	ld	r10, VCPU_KVM(r10)
	lwz	r10, KVM_HOST_LPID(r10)
	mtspr	SPRN_LPID,r10

	ld	r10, HSTATE_KVM_VCPU(r13)
	ld	r10, VCPU_KVM(r10)
	ld	r10, KVM_HOST_LPCR(r10)
	mtspr	SPRN_LPCR,r10

	isync

	/*
	 * Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear
	 * MSR_RI in r12 ([H]SRR1) so the handler won't try to return.
	 */
	li	r10,KVM_GUEST_MODE_NONE
	stb	r10,HSTATE_IN_GUEST(r13)
	li	r10,MSR_RI
	andc	r12,r12,r10

	/*
	 * Go back to interrupt handler. MCE and SRESET have their specific
	 * PACA save area so they should be used directly. They set up their
	 * own stack. The other handlers all use EXGEN. They will use the
	 * guest r1 if it looks like a kernel stack, so just load the
	 * emergency stack and go to program check for all other interrupts.
	 */
	ld	r10,HSTATE_SCRATCH0(r13)
	cmpwi	r10,BOOK3S_INTERRUPT_MACHINE_CHECK
	beq	.Lcall_machine_check_common

	cmpwi	r10,BOOK3S_INTERRUPT_SYSTEM_RESET
	beq	.Lcall_system_reset_common

	b	.

.Lcall_machine_check_common:
	b	machine_check_common

.Lcall_system_reset_common:
	b	system_reset_common
#endif