summaryrefslogtreecommitdiffstats
path: root/arch/arm/kernel/phys2virt.S
blob: fb53db78fe7850bac6eaa83819102ccca115f43b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  Copyright (C) 1994-2002 Russell King
 *  Copyright (c) 2003, 2020 ARM Limited
 *  All Rights Reserved
 */

#include <linux/init.h>
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/page.h>

#ifdef __ARMEB__
#define LOW_OFFSET	0x4
#define HIGH_OFFSET	0x0
#else
#define LOW_OFFSET	0x0
#define HIGH_OFFSET	0x4
#endif

/*
 * __fixup_pv_table - patch the stub instructions with the delta between
 *                    PHYS_OFFSET and PAGE_OFFSET, which is assumed to be
 *                    2 MiB aligned.
 *
 * Called from head.S, which expects the following registers to be preserved:
 *   r1 = machine no, r2 = atags or dtb,
 *   r8 = phys_offset, r9 = cpuid, r10 = procinfo
 */
	__HEAD
ENTRY(__fixup_pv_table)
	mov	r0, r8, lsr #PAGE_SHIFT	@ convert to PFN
	str_l	r0, __pv_phys_pfn_offset, r3

	adr_l	r0, __pv_offset
	subs	r3, r8, #PAGE_OFFSET	@ PHYS_OFFSET - PAGE_OFFSET
	mvn	ip, #0
	strcc	ip, [r0, #HIGH_OFFSET]	@ save to __pv_offset high bits
	str	r3, [r0, #LOW_OFFSET]	@ save to __pv_offset low bits

	mov	r0, r3, lsr #21		@ constant for add/sub instructions
	teq	r3, r0, lsl #21 	@ must be 2 MiB aligned
	bne	0f

	adr_l	r4, __pv_table_begin
	adr_l	r5, __pv_table_end
	b	__fixup_a_pv_table

0:	mov	r0, r0			@ deadloop on error
	b	0b
ENDPROC(__fixup_pv_table)

	.text
__fixup_a_pv_table:
	adr_l	r6, __pv_offset
	ldr	r0, [r6, #HIGH_OFFSET]	@ pv_offset high word
	ldr	r6, [r6, #LOW_OFFSET]	@ pv_offset low word
	cmn	r0, #1
#ifdef CONFIG_THUMB2_KERNEL
	@
	@ The Thumb-2 versions of the patchable sequences are
	@
	@ phys-to-virt:			movw	<reg>, #offset<31:21>
	@				lsl	<reg>, #21
	@				sub	<VA>, <PA>, <reg>
	@
	@ virt-to-phys (non-LPAE):	movw	<reg>, #offset<31:21>
	@				lsl	<reg>, #21
	@				add	<PA>, <VA>, <reg>
	@
	@ virt-to-phys (LPAE):		movw	<reg>, #offset<31:21>
	@				lsl	<reg>, #21
	@				adds	<PAlo>, <VA>, <reg>
	@				mov	<PAhi>, #offset<39:32>
	@				adc	<PAhi>, <PAhi>, #0
	@
	@ In the non-LPAE case, all patchable instructions are MOVW
	@ instructions, where we need to patch in the offset into the
	@ second halfword of the opcode (the 16-bit immediate is encoded
	@ as imm4:i:imm3:imm8)
	@
	@       15       11 10  9           4 3    0  15  14  12 11 8 7    0
	@      +-----------+---+-------------+------++---+------+----+------+
	@ MOVW | 1 1 1 1 0 | i | 1 0 0 1 0 0 | imm4 || 0 | imm3 | Rd | imm8 |
	@      +-----------+---+-------------+------++---+------+----+------+
	@
	@ In the LPAE case, we also need to patch in the high word of the
	@ offset into the immediate field of the MOV instruction, or patch it
	@ to a MVN instruction if the offset is negative. In this case, we
	@ need to inspect the first halfword of the opcode, to check whether
	@ it is MOVW or MOV/MVN, and to perform the MOV to MVN patching if
	@ needed. The encoding of the immediate is rather complex for values
	@ of i:imm3 != 0b0000, but fortunately, we never need more than 8 lower
	@ order bits, which can be patched into imm8 directly (and i:imm3
	@ cleared)
	@
	@      15       11 10  9        5         0  15  14  12 11 8 7    0
	@     +-----------+---+---------------------++---+------+----+------+
	@ MOV | 1 1 1 1 0 | i | 0 0 0 1 0 0 1 1 1 1 || 0 | imm3 | Rd | imm8 |
	@ MVN | 1 1 1 1 0 | i | 0 0 0 1 1 0 1 1 1 1 || 0 | imm3 | Rd | imm8 |
	@     +-----------+---+---------------------++---+------+----+------+
	@
	moveq	r0, #0x200000		@ set bit 21, mov to mvn instruction
	lsrs	r3, r6, #29		@ isolate top 3 bits of displacement
	ubfx	r6, r6, #21, #8		@ put bits 28:21 into the MOVW imm8 field
	bfi	r6, r3, #12, #3		@ put bits 31:29 into the MOVW imm3 field
	b	.Lnext
.Lloop:	add	r7, r4
	adds	r4, #4			@ clears Z flag
#ifdef CONFIG_ARM_LPAE
	ldrh	ip, [r7]
ARM_BE8(rev16	ip, ip)
	tst	ip, #0x200		@ MOVW has bit 9 set, MVN has it clear
	bne	0f			@ skip to MOVW handling (Z flag is clear)
	bic	ip, #0x20		@ clear bit 5 (MVN -> MOV)
	orr	ip, ip, r0, lsr #16	@ MOV -> MVN if offset < 0
ARM_BE8(rev16	ip, ip)
	strh	ip, [r7]
	@ Z flag is set
0:
#endif
	ldrh	ip, [r7, #2]
ARM_BE8(rev16	ip, ip)
	and	ip, #0xf00		@ clear everything except Rd field
	orreq	ip, r0			@ Z flag set -> MOV/MVN -> patch in high bits
	orrne	ip, r6			@ Z flag clear -> MOVW -> patch in low bits
ARM_BE8(rev16	ip, ip)
	strh	ip, [r7, #2]
#else
#ifdef CONFIG_CPU_ENDIAN_BE8
@ in BE8, we load data in BE, but instructions still in LE
#define PV_BIT24	0x00000001
#define PV_IMM8_MASK	0xff000000
#define PV_IMMR_MSB	0x00080000
#else
#define PV_BIT24	0x01000000
#define PV_IMM8_MASK	0x000000ff
#define PV_IMMR_MSB	0x00000800
#endif

	@
	@ The ARM versions of the patchable sequences are
	@
	@ phys-to-virt:			sub	<VA>, <PA>, #offset<31:24>, lsl #24
	@				sub	<VA>, <PA>, #offset<23:16>, lsl #16
	@
	@ virt-to-phys (non-LPAE):	add	<PA>, <VA>, #offset<31:24>, lsl #24
	@				add	<PA>, <VA>, #offset<23:16>, lsl #16
	@
	@ virt-to-phys (LPAE):		movw	<reg>, #offset<31:20>
	@				adds	<PAlo>, <VA>, <reg>, lsl #20
	@				mov	<PAhi>, #offset<39:32>
	@				adc	<PAhi>, <PAhi>, #0
	@
	@ In the non-LPAE case, all patchable instructions are ADD or SUB
	@ instructions, where we need to patch in the offset into the
	@ immediate field of the opcode, which is emitted with the correct
	@ rotation value. (The effective value of the immediate is imm12<7:0>
	@ rotated right by [2 * imm12<11:8>] bits)
	@
	@      31   28 27      23 22  20 19  16 15  12 11    0
	@      +------+-----------------+------+------+-------+
	@  ADD | cond | 0 0 1 0 1 0 0 0 |  Rn  |  Rd  | imm12 |
	@  SUB | cond | 0 0 1 0 0 1 0 0 |  Rn  |  Rd  | imm12 |
	@  MOV | cond | 0 0 1 1 1 0 1 0 |  Rn  |  Rd  | imm12 |
	@  MVN | cond | 0 0 1 1 1 1 1 0 |  Rn  |  Rd  | imm12 |
	@      +------+-----------------+------+------+-------+
	@
	@ In the LPAE case, we use a MOVW instruction to carry the low offset
	@ word, and patch in the high word of the offset into the immediate
	@ field of the subsequent MOV instruction, or patch it to a MVN
	@ instruction if the offset is negative. We can distinguish MOVW
	@ instructions based on bits 23:22 of the opcode, and ADD/SUB can be
	@ distinguished from MOV/MVN (all using the encodings above) using
	@ bit 24.
	@
	@      31   28 27      23 22  20 19  16 15  12 11    0
	@      +------+-----------------+------+------+-------+
	@ MOVW | cond | 0 0 1 1 0 0 0 0 | imm4 |  Rd  | imm12 |
	@      +------+-----------------+------+------+-------+
	@
	moveq	r0, #0x400000		@ set bit 22, mov to mvn instruction
	mov	r3, r6, lsr #16		@ put offset bits 31-16 into r3
	mov	r6, r6, lsr #24		@ put offset bits 31-24 into r6
	and	r3, r3, #0xf0		@ only keep offset bits 23-20 in r3
	b	.Lnext
.Lloop:	ldr	ip, [r7, r4]
#ifdef CONFIG_ARM_LPAE
	tst	ip, #PV_BIT24		@ ADD/SUB have bit 24 clear
	beq	1f
ARM_BE8(rev	ip, ip)
	tst	ip, #0xc00000		@ MOVW has bits 23:22 clear
	bic	ip, ip, #0x400000	@ clear bit 22
	bfc	ip, #0, #12		@ clear imm12 field of MOV[W] instruction
	orreq	ip, ip, r6, lsl #4	@ MOVW -> mask in offset bits 31-24
	orreq	ip, ip, r3, lsr #4	@ MOVW -> mask in offset bits 23-20
	orrne	ip, ip, r0		@ MOV  -> mask in offset bits 7-0 (or bit 22)
ARM_BE8(rev	ip, ip)
	b	2f
1:
#endif
	tst	ip, #PV_IMMR_MSB		@ rotation value >= 16 ?
	bic	ip, ip, #PV_IMM8_MASK
	orreq	ip, ip, r6 ARM_BE8(, lsl #24)	@ mask in offset bits 31-24
	orrne	ip, ip, r3 ARM_BE8(, lsl #24)	@ mask in offset bits 23-20
2:
	str	ip, [r7, r4]
	add	r4, r4, #4
#endif

.Lnext:
	cmp	r4, r5
	ldrcc	r7, [r4]		@ use branch for delay slot
	bcc	.Lloop
	ret	lr
ENDPROC(__fixup_a_pv_table)

ENTRY(fixup_pv_table)
	stmfd	sp!, {r4 - r7, lr}
	mov	r4, r0			@ r0 = table start
	add	r5, r0, r1		@ r1 = table size
	bl	__fixup_a_pv_table
	ldmfd	sp!, {r4 - r7, pc}
ENDPROC(fixup_pv_table)

	.data
	.align	2
	.globl	__pv_phys_pfn_offset
	.type	__pv_phys_pfn_offset, %object
__pv_phys_pfn_offset:
	.word	0
	.size	__pv_phys_pfn_offset, . -__pv_phys_pfn_offset

	.globl	__pv_offset
	.type	__pv_offset, %object
__pv_offset:
	.quad	0
	.size	__pv_offset, . -__pv_offset