summaryrefslogtreecommitdiffstats
path: root/OvmfPkg/IoMmuDxe/IoMmuBuffer.c
blob: 103003cae376b93f156a6cece68f7963316d1bff (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
/** @file

  Copyright (c) 2022, Intel Corporation. All rights reserved.<BR>

  SPDX-License-Identifier: BSD-2-Clause-Patent

**/
#include <Library/BaseLib.h>
#include <Library/BaseMemoryLib.h>
#include <Library/DebugLib.h>
#include <Library/MemoryAllocationLib.h>
#include <Library/MemEncryptSevLib.h>
#include <Library/MemEncryptTdxLib.h>
#include <Library/PcdLib.h>
#include <Library/UefiBootServicesTableLib.h>
#include "IoMmuInternal.h"

extern BOOLEAN  mReservedSharedMemSupported;

#define SIZE_OF_MEM_RANGE(MemRange)  (MemRange->HeaderSize + MemRange->DataSize)

#define RESERVED_MEM_BITMAP_4K_MASK    0xf
#define RESERVED_MEM_BITMAP_32K_MASK   0xff0
#define RESERVED_MEM_BITMAP_128K_MASK  0x3000
#define RESERVED_MEM_BITMAP_1M_MASK    0x40000
#define RESERVED_MEM_BITMAP_2M_MASK    0x180000
#define RESERVED_MEM_BITMAP_MASK       0x1fffff

/**
 * mReservedMemRanges describes the layout of the reserved memory.
 * The reserved memory consists of disfferent size of memory region.
 * The pieces of memory with the same size are managed by one entry
 * in the mReservedMemRanges. All the pieces of memories are managed by
 * mReservedMemBitmap which is a UINT32. It means it can manage at most
 * 32 pieces of memory. Because of the layout of CommonBuffer
 * (1-page header + n-page data), a piece of reserved memory consists of
 * 2 parts: Header + Data.
 *
 * So put all these together, mReservedMemRanges and mReservedMemBitmap
 * are designed to manage the reserved memory.
 *
 * Use the second entry of mReservedMemRanges as an example.
 * { RESERVED_MEM_BITMAP_32K_MASK,  4,  8, SIZE_32KB,  SIZE_4KB, 0 },
 * - RESERVED_MEM_BITMAP_32K_MASK is 0xff0. It means bit4-11 in mReservedMemBitmap
 *   is reserved for 32K size memory.
 * - 4 is the shift of mReservedMemBitmap.
 * - 8 means there are 8 pieces of 32K size memory.
 * - SIZE_32KB indicates the size of Data part.
 * - SIZE_4KB is the size of Header part.
 * - 0 is the start address of this memory range which will be populated when
 *   the reserved memory is initialized.
 *
 * The size and count of the memory region are derived from the experience. For
 * a typical grub boot, there are about 5100 IoMmu/DMA operation. Most of these
 * DMA operation require the memory with size less than 32K (~5080). But we find
 * in grub boot there may be 2 DMA operation which require for the memory larger
 * than 1M. And these 2 DMA operation occur concurrently. So we reserve 2 pieces
 * of memory with size of SIZE_2MB. This is for the best boot performance.
 *
 * If all the reserved memory are exausted, then it will fall back to the legacy
 * memory allocation as before.
 */
STATIC IOMMU_RESERVED_MEM_RANGE  mReservedMemRanges[] = {
  { RESERVED_MEM_BITMAP_4K_MASK,   0,  4, SIZE_4KB,   SIZE_4KB, 0 },
  { RESERVED_MEM_BITMAP_32K_MASK,  4,  8, SIZE_32KB,  SIZE_4KB, 0 },
  { RESERVED_MEM_BITMAP_128K_MASK, 12, 2, SIZE_128KB, SIZE_4KB, 0 },
  { RESERVED_MEM_BITMAP_1M_MASK,   14, 1, SIZE_1MB,   SIZE_4KB, 0 },
  { RESERVED_MEM_BITMAP_2M_MASK,   15, 2, SIZE_2MB,   SIZE_4KB, 0 },
};

//
// Bitmap of the allocation of reserved memory.
//
STATIC UINT32  mReservedMemBitmap = 0;

//
// Start address of the reserved memory region.
//
STATIC EFI_PHYSICAL_ADDRESS  mReservedSharedMemAddress = 0;

//
// Total size of the reserved memory region.
//
STATIC UINT32  mReservedSharedMemSize = 0;

/**
 * Calculate the size of reserved memory.
 *
 * @retval UINT32   Size of the reserved memory
 */
STATIC
UINT32
CalcuateReservedMemSize (
  VOID
  )
{
  UINT32                    Index;
  IOMMU_RESERVED_MEM_RANGE  *MemRange;

  if (mReservedSharedMemSize != 0) {
    return mReservedSharedMemSize;
  }

  for (Index = 0; Index < ARRAY_SIZE (mReservedMemRanges); Index++) {
    MemRange                = &mReservedMemRanges[Index];
    mReservedSharedMemSize += (SIZE_OF_MEM_RANGE (MemRange) * MemRange->Slots);
  }

  return mReservedSharedMemSize;
}

/**
 * Allocate a memory region and convert it to be shared. This memory region will be
 * used in the DMA operation.
 *
 * The pre-alloc memory contains pieces of memory regions with different size. The
 * allocation of the shared memory regions are indicated by a 32-bit bitmap (mReservedMemBitmap).
 *
 * The memory regions are consumed by IoMmuAllocateBuffer (in which CommonBuffer is allocated) and
 * IoMmuMap (in which bounce buffer is allocated).
 *
 * The CommonBuffer contains 2 parts, one page for CommonBufferHeader which is private memory,
 * the other part is shared memory. So the layout of a piece of memory region after initialization
 * looks like:
 *
 *     |------------|----------------------------|
 *     | Header     |    Data                    |  <-- a piece of pre-alloc memory region
 *     | 4k, private| 4k/32k/128k/etc, shared    |
 *     |-----------------------------------------|
 *
 * @retval EFI_SUCCESS      Successfully initialize the reserved memory.
 * @retval EFI_UNSUPPORTED  This feature is not supported.
 */
EFI_STATUS
IoMmuInitReservedSharedMem (
  VOID
  )
{
  EFI_STATUS                Status;
  UINT32                    Index1, Index2;
  UINTN                     TotalPages;
  IOMMU_RESERVED_MEM_RANGE  *MemRange;
  EFI_PHYSICAL_ADDRESS      PhysicalAddress;
  UINT64                    SharedAddress;

  if (!mReservedSharedMemSupported) {
    return EFI_UNSUPPORTED;
  }

  TotalPages = EFI_SIZE_TO_PAGES (CalcuateReservedMemSize ());

  PhysicalAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)AllocatePages (TotalPages);
  DEBUG ((
    DEBUG_VERBOSE,
    "%a: ReservedMem (%d pages) address = 0x%llx\n",
    __func__,
    TotalPages,
    PhysicalAddress
    ));

  mReservedMemBitmap        = 0;
  mReservedSharedMemAddress = PhysicalAddress;

  for (Index1 = 0; Index1 < ARRAY_SIZE (mReservedMemRanges); Index1++) {
    MemRange                         = &mReservedMemRanges[Index1];
    MemRange->StartAddressOfMemRange = PhysicalAddress;

    for (Index2 = 0; Index2 < MemRange->Slots; Index2++) {
      SharedAddress = (UINT64)(UINTN)(MemRange->StartAddressOfMemRange + Index2 * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize);

      if (CC_GUEST_IS_SEV (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
        Status = MemEncryptSevClearPageEncMask (
                   0,
                   SharedAddress,
                   EFI_SIZE_TO_PAGES (MemRange->DataSize)
                   );
        ASSERT (!EFI_ERROR (Status));
      } else if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
        Status = MemEncryptTdxSetPageSharedBit (
                   0,
                   SharedAddress,
                   EFI_SIZE_TO_PAGES (MemRange->DataSize)
                   );
        ASSERT (!EFI_ERROR (Status));
      } else {
        ASSERT (FALSE);
      }
    }

    PhysicalAddress += (MemRange->Slots * SIZE_OF_MEM_RANGE (MemRange));
  }

  return EFI_SUCCESS;
}

/**
 * Release the pre-alloc shared memory.
 *
 * @retval EFI_SUCCESS  Successfully release the shared memory
 */
EFI_STATUS
IoMmuReleaseReservedSharedMem (
  BOOLEAN  MemoryMapLocked
  )
{
  EFI_STATUS                Status;
  UINT32                    Index1, Index2;
  IOMMU_RESERVED_MEM_RANGE  *MemRange;
  UINT64                    SharedAddress;

  if (!mReservedSharedMemSupported) {
    return EFI_SUCCESS;
  }

  for (Index1 = 0; Index1 < ARRAY_SIZE (mReservedMemRanges); Index1++) {
    MemRange = &mReservedMemRanges[Index1];
    for (Index2 = 0; Index2 < MemRange->Slots; Index2++) {
      SharedAddress = (UINT64)(UINTN)(MemRange->StartAddressOfMemRange + Index2 * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize);

      if (CC_GUEST_IS_SEV (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
        Status = MemEncryptSevSetPageEncMask (
                   0,
                   SharedAddress,
                   EFI_SIZE_TO_PAGES (MemRange->DataSize)
                   );
        ASSERT (!EFI_ERROR (Status));
      } else if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
        Status = MemEncryptTdxClearPageSharedBit (
                   0,
                   SharedAddress,
                   EFI_SIZE_TO_PAGES (MemRange->DataSize)
                   );
        ASSERT (!EFI_ERROR (Status));
      } else {
        ASSERT (FALSE);
      }
    }
  }

  if (!MemoryMapLocked) {
    FreePages ((VOID *)(UINTN)mReservedSharedMemAddress, EFI_SIZE_TO_PAGES (CalcuateReservedMemSize ()));
    mReservedSharedMemAddress = 0;
    mReservedMemBitmap        = 0;
  }

  mReservedSharedMemSupported = FALSE;

  return EFI_SUCCESS;
}

/**
 * Allocate from the reserved memory pool.
 * If the reserved shared memory is exausted or there is no suitalbe size, it turns
 * to the LegacyAllocateBuffer.
 *
 * @param Type                Allocate type
 * @param MemoryType          The memory type to be allocated
 * @param Pages               Pages to be allocated.
 * @param ReservedMemBitmap   Bitmap of the allocated memory region
 * @param PhysicalAddress     Pointer to the data part of allocated memory region
 *
 * @retval EFI_SUCCESS        Successfully allocate the buffer
 * @retval Other              As the error code indicates
 */
STATIC
EFI_STATUS
InternalAllocateBuffer (
  IN  EFI_ALLOCATE_TYPE        Type,
  IN  EFI_MEMORY_TYPE          MemoryType,
  IN  UINTN                    Pages,
  IN OUT UINT32                *ReservedMemBitmap,
  IN OUT EFI_PHYSICAL_ADDRESS  *PhysicalAddress
  )
{
  UINT32                    MemBitmap;
  UINT8                     Index;
  IOMMU_RESERVED_MEM_RANGE  *MemRange;
  UINTN                     PagesOfLastMemRange;

  *ReservedMemBitmap = 0;

  if (Pages == 0) {
    ASSERT (FALSE);
    return EFI_INVALID_PARAMETER;
  }

  if (!mReservedSharedMemSupported) {
    goto LegacyAllocateBuffer;
  }

  if (mReservedSharedMemAddress == 0) {
    goto LegacyAllocateBuffer;
  }

  PagesOfLastMemRange = 0;

  for (Index = 0; Index < ARRAY_SIZE (mReservedMemRanges); Index++) {
    if ((Pages > PagesOfLastMemRange) && (Pages <= EFI_SIZE_TO_PAGES (mReservedMemRanges[Index].DataSize))) {
      break;
    }

    PagesOfLastMemRange = EFI_SIZE_TO_PAGES (mReservedMemRanges[Index].DataSize);
  }

  if (Index == ARRAY_SIZE (mReservedMemRanges)) {
    // There is no suitable size of reserved memory. Turn to legacy allocate.
    goto LegacyAllocateBuffer;
  }

  MemRange = &mReservedMemRanges[Index];

  if ((mReservedMemBitmap & MemRange->BitmapMask) == MemRange->BitmapMask) {
    // The reserved memory is exausted. Turn to legacy allocate.
    goto LegacyAllocateBuffer;
  }

  MemBitmap = (mReservedMemBitmap & MemRange->BitmapMask) >> MemRange->Shift;

  for (Index = 0; Index < MemRange->Slots; Index++) {
    if ((MemBitmap & (UINT8)(1<<Index)) == 0) {
      break;
    }
  }

  ASSERT (Index != MemRange->Slots);

  *PhysicalAddress   = MemRange->StartAddressOfMemRange + Index * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize;
  *ReservedMemBitmap = (UINT32)(1 << (Index + MemRange->Shift));

  DEBUG ((
    DEBUG_VERBOSE,
    "%a: range-size: %lx, start-address=0x%llx, pages=0x%llx, bits=0x%lx, bitmap: %lx => %lx\n",
    __func__,
    MemRange->DataSize,
    *PhysicalAddress,
    Pages,
    *ReservedMemBitmap,
    mReservedMemBitmap,
    mReservedMemBitmap | *ReservedMemBitmap
    ));

  return EFI_SUCCESS;

LegacyAllocateBuffer:

  *ReservedMemBitmap = 0;
  return gBS->AllocatePages (Type, MemoryType, Pages, PhysicalAddress);
}

/**
 * Allocate reserved shared memory for bounce buffer.
 *
 * @param Type        Allocate type
 * @param MemoryType  The memory type to be allocated
 * @param MapInfo     Pointer to the MAP_INFO
 *
 * @retval EFI_SUCCESS        Successfully allocate the bounce buffer
 * @retval Other              As the error code indicates

 */
EFI_STATUS
IoMmuAllocateBounceBuffer (
  IN     EFI_ALLOCATE_TYPE  Type,
  IN     EFI_MEMORY_TYPE    MemoryType,
  IN OUT MAP_INFO           *MapInfo
  )
{
  EFI_STATUS  Status;
  UINT32      ReservedMemBitmap;
  EFI_TPL     OldTpl;

  OldTpl            = gBS->RaiseTPL (TPL_NOTIFY);
  ReservedMemBitmap = 0;
  Status            = InternalAllocateBuffer (
                        Type,
                        MemoryType,
                        MapInfo->NumberOfPages,
                        &ReservedMemBitmap,
                        &MapInfo->PlainTextAddress
                        );
  MapInfo->ReservedMemBitmap = ReservedMemBitmap;
  mReservedMemBitmap        |= ReservedMemBitmap;
  gBS->RestoreTPL (OldTpl);

  ASSERT (Status == EFI_SUCCESS);

  return Status;
}

/**
 * Free the bounce buffer allocated in IoMmuAllocateBounceBuffer.
 *
 * @param MapInfo       Pointer to the MAP_INFO
 * @return EFI_SUCCESS  Successfully free the bounce buffer.
 */
EFI_STATUS
IoMmuFreeBounceBuffer (
  IN OUT     MAP_INFO  *MapInfo
  )
{
  EFI_TPL  OldTpl;

  if (MapInfo->ReservedMemBitmap == 0) {
    gBS->FreePages (MapInfo->PlainTextAddress, MapInfo->NumberOfPages);
  } else {
    DEBUG ((
      DEBUG_VERBOSE,
      "%a: PlainTextAddress=0x%Lx, bits=0x%Lx, bitmap: %Lx => %Lx\n",
      __func__,
      MapInfo->PlainTextAddress,
      MapInfo->ReservedMemBitmap,
      mReservedMemBitmap,
      mReservedMemBitmap & ((UINT32)(~MapInfo->ReservedMemBitmap))
      ));
    OldTpl                     = gBS->RaiseTPL (TPL_NOTIFY);
    MapInfo->PlainTextAddress  = 0;
    mReservedMemBitmap        &= (UINT32)(~MapInfo->ReservedMemBitmap);
    MapInfo->ReservedMemBitmap = 0;
    gBS->RestoreTPL (OldTpl);
  }

  return EFI_SUCCESS;
}

/**
 * Allocate CommonBuffer from pre-allocated shared memory.
 *
 * @param MemoryType          Memory type
 * @param CommonBufferPages   Pages of CommonBuffer
 * @param PhysicalAddress     Allocated physical address
 * @param ReservedMemBitmap   Bitmap which indicates the allocation of reserved memory
 *
 * @retval EFI_SUCCESS        Successfully allocate the common buffer
 * @retval Other              As the error code indicates
 */
EFI_STATUS
IoMmuAllocateCommonBuffer (
  IN EFI_MEMORY_TYPE        MemoryType,
  IN UINTN                  CommonBufferPages,
  OUT EFI_PHYSICAL_ADDRESS  *PhysicalAddress,
  OUT UINT32                *ReservedMemBitmap
  )
{
  EFI_STATUS  Status;

  Status = InternalAllocateBuffer (
             AllocateMaxAddress,
             MemoryType,
             CommonBufferPages,
             ReservedMemBitmap,
             PhysicalAddress
             );
  ASSERT (Status == EFI_SUCCESS);

  mReservedMemBitmap |= *ReservedMemBitmap;

  if (*ReservedMemBitmap != 0) {
    *PhysicalAddress -= SIZE_4KB;
  }

  return Status;
}

/**
 * Free CommonBuffer which is allocated by IoMmuAllocateCommonBuffer().
 *
 * @param CommonBufferHeader  Pointer to the CommonBufferHeader
 * @param CommonBufferPages   Pages of CommonBuffer
 *
 * @retval EFI_SUCCESS        Successfully free the common buffer
 * @retval Other              As the error code indicates
 */
EFI_STATUS
IoMmuFreeCommonBuffer (
  IN COMMON_BUFFER_HEADER  *CommonBufferHeader,
  IN UINTN                 CommonBufferPages
  )
{
  if (!mReservedSharedMemSupported) {
    goto LegacyFreeCommonBuffer;
  }

  if (CommonBufferHeader->ReservedMemBitmap == 0) {
    goto LegacyFreeCommonBuffer;
  }

  DEBUG ((
    DEBUG_VERBOSE,
    "%a: CommonBuffer=0x%Lx, bits=0x%Lx, bitmap: %Lx => %Lx\n",
    __func__,
    (UINT64)(UINTN)CommonBufferHeader + SIZE_4KB,
    CommonBufferHeader->ReservedMemBitmap,
    mReservedMemBitmap,
    mReservedMemBitmap & ((UINT32)(~CommonBufferHeader->ReservedMemBitmap))
    ));

  mReservedMemBitmap &= (UINT32)(~CommonBufferHeader->ReservedMemBitmap);
  return EFI_SUCCESS;

LegacyFreeCommonBuffer:
  return gBS->FreePages ((UINTN)CommonBufferHeader, CommonBufferPages);
}