summaryrefslogtreecommitdiffstats
path: root/arch/arm/common/mcpm_head.S
blob: 291d969bc719cdd77c74c5f3e1735f7d0fe56a1d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM
 *
 * Created by:  Nicolas Pitre, March 2012
 * Copyright:   (C) 2012-2013  Linaro Limited
 *
 * Refer to Documentation/arm/cluster-pm-race-avoidance.rst
 * for details of the synchronisation algorithms used here.
 */

#include <linux/linkage.h>
#include <asm/mcpm.h>
#include <asm/assembler.h>

#include "vlock.h"

.if MCPM_SYNC_CLUSTER_CPUS
.error "cpus must be the first member of struct mcpm_sync_struct"
.endif

	.macro	pr_dbg	string
#if defined(CONFIG_DEBUG_LL) && defined(DEBUG)
	b	1901f
1902:	.asciz	"CPU"
1903:	.asciz	" cluster"
1904:	.asciz	": \string"
	.align
1901:	adr	r0, 1902b
	bl	printascii
	mov	r0, r9
	bl	printhex2
	adr	r0, 1903b
	bl	printascii
	mov	r0, r10
	bl	printhex2
	adr	r0, 1904b
	bl	printascii
#endif
	.endm

	.arm
	.align

ENTRY(mcpm_entry_point)

 ARM_BE8(setend        be)
 THUMB(	badr	r12, 1f		)
 THUMB(	bx	r12		)
 THUMB(	.thumb			)
1:
	mrc	p15, 0, r0, c0, c0, 5		@ MPIDR
	ubfx	r9, r0, #0, #8			@ r9 = cpu
	ubfx	r10, r0, #8, #8			@ r10 = cluster
	mov	r3, #MAX_CPUS_PER_CLUSTER
	mla	r4, r3, r10, r9			@ r4 = canonical CPU index
	cmp	r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS)
	blo	2f

	/* We didn't expect this CPU.  Try to cheaply make it quiet. */
1:	wfi
	wfe
	b	1b

2:	pr_dbg	"kernel mcpm_entry_point\n"

	/*
	 * MMU is off so we need to get to various variables in a
	 * position independent way.
	 */
	adr	r5, 3f
	ldmia	r5, {r0, r6, r7, r8, r11}
	add	r0, r5, r0			@ r0 = mcpm_entry_early_pokes
	add	r6, r5, r6			@ r6 = mcpm_entry_vectors
	ldr	r7, [r5, r7]			@ r7 = mcpm_power_up_setup_phys
	add	r8, r5, r8			@ r8 = mcpm_sync
	add	r11, r5, r11			@ r11 = first_man_locks

	@ Perform an early poke, if any
	add	r0, r0, r4, lsl #3
	ldmia	r0, {r0, r1}
	teq	r0, #0
	strne	r1, [r0]

	mov	r0, #MCPM_SYNC_CLUSTER_SIZE
	mla	r8, r0, r10, r8			@ r8 = sync cluster base

	@ Signal that this CPU is coming UP:
	mov	r0, #CPU_COMING_UP
	mov	r5, #MCPM_SYNC_CPU_SIZE
	mla	r5, r9, r5, r8			@ r5 = sync cpu address
	strb	r0, [r5]

	@ At this point, the cluster cannot unexpectedly enter the GOING_DOWN
	@ state, because there is at least one active CPU (this CPU).

	mov	r0, #VLOCK_SIZE
	mla	r11, r0, r10, r11		@ r11 = cluster first man lock
	mov	r0, r11
	mov	r1, r9				@ cpu
	bl	vlock_trylock			@ implies DMB

	cmp	r0, #0				@ failed to get the lock?
	bne	mcpm_setup_wait		@ wait for cluster setup if so

	ldrb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	cmp	r0, #CLUSTER_UP			@ cluster already up?
	bne	mcpm_setup			@ if not, set up the cluster

	@ Otherwise, release the first man lock and skip setup:
	mov	r0, r11
	bl	vlock_unlock
	b	mcpm_setup_complete

mcpm_setup:
	@ Control dependency implies strb not observable before previous ldrb.

	@ Signal that the cluster is being brought up:
	mov	r0, #INBOUND_COMING_UP
	strb	r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
	dmb

	@ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this
	@ point onwards will observe INBOUND_COMING_UP and abort.

	@ Wait for any previously-pending cluster teardown operations to abort
	@ or complete:
mcpm_teardown_wait:
	ldrb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	cmp	r0, #CLUSTER_GOING_DOWN
	bne	first_man_setup
	wfe
	b	mcpm_teardown_wait

first_man_setup:
	dmb

	@ If the outbound gave up before teardown started, skip cluster setup:

	cmp	r0, #CLUSTER_UP
	beq	mcpm_setup_leave

	@ power_up_setup is now responsible for setting up the cluster:

	cmp	r7, #0
	mov	r0, #1		@ second (cluster) affinity level
	blxne	r7		@ Call power_up_setup if defined
	dmb

	mov	r0, #CLUSTER_UP
	strb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	dmb

mcpm_setup_leave:
	@ Leave the cluster setup critical section:

	mov	r0, #INBOUND_NOT_COMING_UP
	strb	r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
	dsb	st
	sev

	mov	r0, r11
	bl	vlock_unlock	@ implies DMB
	b	mcpm_setup_complete

	@ In the contended case, non-first men wait here for cluster setup
	@ to complete:
mcpm_setup_wait:
	ldrb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	cmp	r0, #CLUSTER_UP
	wfene
	bne	mcpm_setup_wait
	dmb

mcpm_setup_complete:
	@ If a platform-specific CPU setup hook is needed, it is
	@ called from here.

	cmp	r7, #0
	mov	r0, #0		@ first (CPU) affinity level
	blxne	r7		@ Call power_up_setup if defined
	dmb

	@ Mark the CPU as up:

	mov	r0, #CPU_UP
	strb	r0, [r5]

	@ Observability order of CPU_UP and opening of the gate does not matter.

mcpm_entry_gated:
	ldr	r5, [r6, r4, lsl #2]		@ r5 = CPU entry vector
	cmp	r5, #0
	wfeeq
	beq	mcpm_entry_gated
	dmb

	pr_dbg	"released\n"
	bx	r5

	.align	2

3:	.word	mcpm_entry_early_pokes - .
	.word	mcpm_entry_vectors - 3b
	.word	mcpm_power_up_setup_phys - 3b
	.word	mcpm_sync - 3b
	.word	first_man_locks - 3b

ENDPROC(mcpm_entry_point)

	.bss

	.align	CACHE_WRITEBACK_ORDER
	.type	first_man_locks, #object
first_man_locks:
	.space	VLOCK_SIZE * MAX_NR_CLUSTERS
	.align	CACHE_WRITEBACK_ORDER

	.type	mcpm_entry_vectors, #object
ENTRY(mcpm_entry_vectors)
	.space	4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER

	.type	mcpm_entry_early_pokes, #object
ENTRY(mcpm_entry_early_pokes)
	.space	8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER

	.type	mcpm_power_up_setup_phys, #object
ENTRY(mcpm_power_up_setup_phys)
	.space  4		@ set by mcpm_sync_init()