1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
|
;------------------------------------------------------------------------------ ;
; Copyright (c) 2015 - 2023, Intel Corporation. All rights reserved.<BR>
; SPDX-License-Identifier: BSD-2-Clause-Patent
;
; Module Name:
;
; MpFuncs.nasm
;
; Abstract:
;
; This is the assembly code for MP support
;
;-------------------------------------------------------------------------------
%include "MpEqu.inc"
extern ASM_PFX(InitializeFloatingPointUnits)
%macro OneTimeCall 1
jmp %1
%1 %+ OneTimerCallReturn:
%endmacro
%macro OneTimeCallRet 1
jmp %1 %+ OneTimerCallReturn
%endmacro
DEFAULT REL
SECTION .text
;-------------------------------------------------------------------------------------
;RendezvousFunnelProc procedure follows. All APs execute their procedure. This
;procedure serializes all the AP processors through an Init sequence. It must be
;noted that APs arrive here very raw...ie: real mode, no stack.
;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC
;IS IN MACHINE CODE.
;-------------------------------------------------------------------------------------
RendezvousFunnelProcStart:
; At this point CS = 0x(vv00) and ip= 0x0.
; Save BIST information to ebp firstly
BITS 16
mov ebp, eax ; Save BIST information
mov ax, cs
mov ds, ax
mov es, ax
mov ss, ax
xor ax, ax
mov fs, ax
mov gs, ax
mov si, MP_CPU_EXCHANGE_INFO_FIELD (BufferStart)
mov ebx, [si]
mov si, MP_CPU_EXCHANGE_INFO_FIELD (DataSegment)
mov edx, [si]
;
; Get start address of 32-bit code in low memory (<1MB)
;
mov edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeTransitionMemory)
mov si, MP_CPU_EXCHANGE_INFO_FIELD (GdtrProfile)
o32 lgdt [cs:si]
;
; Switch to protected mode
;
mov eax, cr0 ; Get control register 0
or eax, 000000003h ; Set PE bit (bit #0) & MP
mov cr0, eax
; Switch to 32-bit code (>1MB)
o32 jmp far [cs:di]
;
; Following code must be copied to memory with type of EfiBootServicesCode.
; This is required if NX is enabled for EfiBootServicesCode of memory.
;
BITS 32
Flat32Start: ; protected mode entry point
mov ds, dx
mov es, dx
mov fs, dx
mov gs, dx
mov ss, dx
;
; Enable execute disable bit
;
mov esi, MP_CPU_EXCHANGE_INFO_FIELD (EnableExecuteDisable)
cmp byte [ebx + esi], 0
jz SkipEnableExecuteDisableBit
mov ecx, 0c0000080h ; EFER MSR number
rdmsr ; Read EFER
bts eax, 11 ; Enable Execute Disable Bit
wrmsr ; Write EFER
SkipEnableExecuteDisableBit:
;
; Enable PAE
;
mov eax, cr4
bts eax, 5
mov esi, MP_CPU_EXCHANGE_INFO_FIELD (Enable5LevelPaging)
cmp byte [ebx + esi], 0
jz SkipEnable5LevelPaging
;
; Enable 5 Level Paging
;
bts eax, 12 ; Set LA57=1.
SkipEnable5LevelPaging:
mov cr4, eax
;
; Load page table
;
mov esi, MP_CPU_EXCHANGE_INFO_FIELD (Cr3) ; Save CR3 in ecx
mov ecx, [ebx + esi]
mov cr3, ecx ; Load CR3
;
; Enable long mode
;
mov ecx, 0c0000080h ; EFER MSR number
rdmsr ; Read EFER
bts eax, 8 ; Set LME=1
wrmsr ; Write EFER
;
; Enable paging
;
mov eax, cr0 ; Read CR0
bts eax, 31 ; Set PG=1
mov cr0, eax ; Write CR0
;
; Far jump to 64-bit code
;
mov edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeHighMemory)
add edi, ebx
jmp far [edi]
BITS 64
LongModeStart:
mov esi, ebx
; Set IDT table at the start of 64 bit code
lea edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (IdtrProfile)]
lidt [edi]
lea edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitFlag)]
cmp qword [edi], 1 ; ApInitConfig
jnz GetApicId
; Increment the number of APs executing here as early as possible
; This is decremented in C code when AP is finished executing
mov edi, esi
add edi, MP_CPU_EXCHANGE_INFO_FIELD (NumApsExecuting)
lock inc dword [edi]
; AP init
mov edi, esi
add edi, MP_CPU_EXCHANGE_INFO_FIELD (ApIndex)
mov ebx, 1
lock xadd dword [edi], ebx ; EBX = ApIndex++
inc ebx ; EBX is CpuNumber
; program stack
mov edi, esi
add edi, MP_CPU_EXCHANGE_INFO_FIELD (StackSize)
mov eax, dword [edi]
mov ecx, ebx
inc ecx
mul ecx ; EAX = StackSize * (CpuNumber + 1)
mov edi, esi
add edi, MP_CPU_EXCHANGE_INFO_FIELD (StackStart)
add rax, qword [edi]
mov rsp, rax
;
; Setup the GHCB when AMD SEV-ES active.
;
OneTimeCall SevEsSetupGhcb
jmp CProcedureInvoke
GetApicId:
;
; Use the GHCB protocol to get the ApicId when SEV-ES is active.
;
OneTimeCall SevEsGetApicId
DoCpuid:
mov eax, 0
cpuid
cmp eax, 0bh
jb NoX2Apic ; CPUID level below CPUID_EXTENDED_TOPOLOGY
mov eax, 0bh
xor ecx, ecx
cpuid
test ebx, 0ffffh
jz NoX2Apic ; CPUID.0BH:EBX[15:0] is zero
; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX
jmp GetProcessorNumber
NoX2Apic:
; Processor is not x2APIC capable, so get 8-bit APIC ID
mov eax, 1
cpuid
shr ebx, 24
mov edx, ebx
GetProcessorNumber:
;
; Get processor number for this AP
; Note that BSP may become an AP due to SwitchBsp()
;
xor ebx, ebx
lea eax, [esi + MP_CPU_EXCHANGE_INFO_FIELD (CpuInfo)]
mov rdi, [eax]
GetNextProcNumber:
cmp dword [rdi + CPU_INFO_IN_HOB.InitialApicId], edx ; APIC ID match?
jz ProgramStack
add rdi, CPU_INFO_IN_HOB_size
inc ebx
jmp GetNextProcNumber
ProgramStack:
mov rsp, qword [rdi + CPU_INFO_IN_HOB.ApTopOfStack]
CProcedureInvoke:
;
; Reserve 8 bytes for CpuMpData.
; When the AP wakes up again via INIT-SIPI-SIPI, push 0 will cause the existing CpuMpData to be overwritten with 0.
; CpuMpData is filled in via InitializeApData() during the first time INIT-SIPI-SIPI,
; while overwirrten may occurs when under ApInHltLoop but InitFlag is not set to ApInitConfig.
; Therefore reservation is implemented by sub rsp instead of push 0.
;
sub rsp, 8
push rbp ; Push BIST data at top of AP stack
xor rbp, rbp ; Clear ebp for call stack trace
push rbp
mov rbp, rsp
push qword 0 ; Push 8 bytes for alignment
mov rax, qword [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitializeFloatingPointUnits)]
sub rsp, 20h
call rax ; Call assembly function to initialize FPU per UEFI spec
add rsp, 20h
mov edx, ebx ; edx is ApIndex
mov ecx, esi
add ecx, MP_CPU_EXCHANGE_INFO_OFFSET ; rcx is address of exchange info data buffer
mov edi, esi
add edi, MP_CPU_EXCHANGE_INFO_FIELD (CFunction)
mov rax, qword [edi]
sub rsp, 20h
call rax ; Invoke C function
add rsp, 20h
jmp $ ; Should never reach here
;
; Required for the AMD SEV helper functions
;
%include "AmdSev.nasm"
RendezvousFunnelProcEnd:
;-------------------------------------------------------------------------------------
; AsmRelocateApLoopAmdSev (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);
;-------------------------------------------------------------------------------------
AsmRelocateApLoopAmdSevStart:
BITS 64
cmp qword [rsp + 56], 0 ; SevEsAPJumpTable
je NoSevEsAmdSev
;
; Perform some SEV-ES related setup before leaving 64-bit mode
;
push rcx
push rdx
;
; Get the RDX reset value using CPUID
;
mov rax, 1
cpuid
mov rsi, rax ; Save off the reset value for RDX
;
; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call
; - Must be done while in 64-bit long mode so that writes to
; the GHCB memory will be unencrypted.
; - No NAE events can be generated once this is set otherwise
; the AP_RESET_HOLD SW_EXITCODE will be overwritten.
;
mov rcx, 0xc0010130
rdmsr ; Retrieve current GHCB address
shl rdx, 32
or rdx, rax
mov rdi, rdx
xor rax, rax
mov rcx, 0x800
shr rcx, 3
rep stosq ; Clear the GHCB
mov rax, 0x80000004 ; VMGEXIT AP_RESET_HOLD
mov [rdx + 0x390], rax
mov rax, 114 ; Set SwExitCode valid bit
bts [rdx + 0x3f0], rax
inc rax ; Set SwExitInfo1 valid bit
bts [rdx + 0x3f0], rax
inc rax ; Set SwExitInfo2 valid bit
bts [rdx + 0x3f0], rax
pop rdx
pop rcx
NoSevEsAmdSev:
cli ; Disable interrupt before switching to 32-bit mode
mov rax, [rsp + 40] ; CountTofinish
lock dec dword [rax] ; (*CountTofinish)--
mov r10, [rsp + 48] ; Pm16CodeSegment
mov rax, [rsp + 56] ; SevEsAPJumpTable
mov rbx, [rsp + 64] ; WakeupBuffer
mov rsp, r9 ; TopOfApStack
push rax ; Save SevEsAPJumpTable
push rbx ; Save WakeupBuffer
push r10 ; Save Pm16CodeSegment
push rcx ; Save MwaitSupport
push rdx ; Save ApTargetCState
lea rax, [PmEntryAmdSev] ; rax <- The start address of transition code
push r8
push rax
;
; Clear R8 - R15, for reset, before going into 32-bit mode
;
xor r8, r8
xor r9, r9
xor r10, r10
xor r11, r11
xor r12, r12
xor r13, r13
xor r14, r14
xor r15, r15
;
; Far return into 32-bit mode
;
o64 retf
BITS 32
PmEntryAmdSev:
mov eax, cr0
btr eax, 31 ; Clear CR0.PG
mov cr0, eax ; Disable paging and caches
mov ecx, 0xc0000080
rdmsr
and ah, ~ 1 ; Clear LME
wrmsr
mov eax, cr4
and al, ~ (1 << 5) ; Clear PAE
mov cr4, eax
pop edx
add esp, 4
pop ecx,
add esp, 4
MwaitCheckAmdSev:
cmp cl, 1 ; Check mwait-monitor support
jnz HltLoopAmdSev
mov ebx, edx ; Save C-State to ebx
MwaitLoopAmdSev:
cli
mov eax, esp ; Set Monitor Address
xor ecx, ecx ; ecx = 0
xor edx, edx ; edx = 0
monitor
mov eax, ebx ; Mwait Cx, Target C-State per eax[7:4]
shl eax, 4
mwait
jmp MwaitLoopAmdSev
HltLoopAmdSev:
pop edx ; PM16CodeSegment
add esp, 4
pop ebx ; WakeupBuffer
add esp, 4
pop eax ; SevEsAPJumpTable
add esp, 4
cmp eax, 0 ; Check for SEV-ES
je DoHltAmdSev
cli
;
; SEV-ES is enabled, use VMGEXIT (GHCB information already
; set by caller)
;
BITS 64
rep vmmcall
BITS 32
;
; Back from VMGEXIT AP_HLT_LOOP
; Push the FLAGS/CS/IP values to use
;
push word 0x0002 ; EFLAGS
xor ecx, ecx
mov cx, [eax + 2] ; CS
push cx
mov cx, [eax] ; IP
push cx
push word 0x0000 ; For alignment, will be discarded
push edx
push ebx
mov edx, esi ; Restore RDX reset value
retf
DoHltAmdSev:
cli
hlt
jmp DoHltAmdSev
BITS 64
AsmRelocateApLoopAmdSevEnd:
;-------------------------------------------------------------------------------------
; AsmRelocateApLoop (MwaitSupport, ApTargetCState, TopOfApStack, CountTofinish, Cr3);
; This function is called during the finalizaiton of Mp initialization before booting
; to OS, and aim to put Aps either in Mwait or HLT.
;-------------------------------------------------------------------------------------
; +----------------+
; | Cr3 | rsp+40
; +----------------+
; | CountTofinish | r9
; +----------------+
; | TopOfApStack | r8
; +----------------+
; | ApTargetCState | rdx
; +----------------+
; | MwaitSupport | rcx
; +----------------+
; | the return |
; +----------------+ low address
AsmRelocateApLoopGenericStart:
mov rax, r9 ; CountTofinish
lock dec dword [rax] ; (*CountTofinish)--
mov rax, [rsp + 40] ; Cr3
; Do not push on old stack, since old stack is not mapped
; in the page table pointed by cr3
mov cr3, rax
mov rsp, r8 ; TopOfApStack
MwaitCheckGeneric:
cmp cl, 1 ; Check mwait-monitor support
jnz HltLoopGeneric
mov rbx, rdx ; Save C-State to ebx
MwaitLoopGeneric:
cli
mov rax, rsp ; Set Monitor Address
xor ecx, ecx ; ecx = 0
xor edx, edx ; edx = 0
monitor
mov rax, rbx ; Mwait Cx, Target C-State per eax[7:4]
shl eax, 4
mwait
jmp MwaitLoopGeneric
HltLoopGeneric:
cli
hlt
jmp HltLoopGeneric
AsmRelocateApLoopGenericEnd:
;-------------------------------------------------------------------------------------
; AsmGetAddressMap (&AddressMap);
;-------------------------------------------------------------------------------------
global ASM_PFX(AsmGetAddressMap)
ASM_PFX(AsmGetAddressMap):
lea rax, [RendezvousFunnelProcStart]
mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelAddress], rax
mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeEntryOffset], LongModeStart - RendezvousFunnelProcStart
mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelSize], RendezvousFunnelProcEnd - RendezvousFunnelProcStart
lea rax, [AsmRelocateApLoopGenericStart]
mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncAddressGeneric], rax
mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncSizeGeneric], AsmRelocateApLoopGenericEnd - AsmRelocateApLoopGenericStart
lea rax, [AsmRelocateApLoopAmdSevStart]
mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncAddressAmdSev], rax
mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncSizeAmdSev], AsmRelocateApLoopAmdSevEnd - AsmRelocateApLoopAmdSevStart
mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeTransitionOffset], Flat32Start - RendezvousFunnelProcStart
mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealNoNxOffset], SwitchToRealProcStart - Flat32Start
mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeOffset], PM16Mode - RendezvousFunnelProcStart
mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeSize], SwitchToRealProcEnd - PM16Mode
ret
;-------------------------------------------------------------------------------------
;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is
;about to become an AP. It switches its stack with the current AP.
;AsmExchangeRole (IN CPU_EXCHANGE_INFO *MyInfo, IN CPU_EXCHANGE_INFO *OthersInfo);
;-------------------------------------------------------------------------------------
global ASM_PFX(AsmExchangeRole)
ASM_PFX(AsmExchangeRole):
; DO NOT call other functions in this function, since 2 CPU may use 1 stack
; at the same time. If 1 CPU try to call a function, stack will be corrupted.
push rax
push rbx
push rcx
push rdx
push rsi
push rdi
push rbp
push r8
push r9
push r10
push r11
push r12
push r13
push r14
push r15
; rsi contains MyInfo pointer
mov rsi, rcx
; rdi contains OthersInfo pointer
mov rdi, rdx
pushfq
; Store the its StackPointer
mov [rsi + CPU_EXCHANGE_ROLE_INFO.StackPointer], rsp
; update its switch state to STORED
mov byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
WaitForOtherStored:
; wait until the other CPU finish storing its state
cmp byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
jz OtherStored
pause
jmp WaitForOtherStored
OtherStored:
; load its future StackPointer
mov rsp, [rdi + CPU_EXCHANGE_ROLE_INFO.StackPointer]
; update the other CPU's switch state to LOADED
mov byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
WaitForOtherLoaded:
; wait until the other CPU finish loading new state,
; otherwise the data in stack may corrupt
cmp byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
jz OtherLoaded
pause
jmp WaitForOtherLoaded
OtherLoaded:
; since the other CPU already get the data it want, leave this procedure
popfq
pop r15
pop r14
pop r13
pop r12
pop r11
pop r10
pop r9
pop r8
pop rbp
pop rdi
pop rsi
pop rdx
pop rcx
pop rbx
pop rax
ret
|