IoMmuBuffer.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495
  1. /** @file
  2. Copyright (c) 2022, Intel Corporation. All rights reserved.<BR>
  3. SPDX-License-Identifier: BSD-2-Clause-Patent
  4. **/
  5. #include <Library/BaseLib.h>
  6. #include <Library/BaseMemoryLib.h>
  7. #include <Library/DebugLib.h>
  8. #include <Library/MemoryAllocationLib.h>
  9. #include <Library/MemEncryptSevLib.h>
  10. #include <Library/MemEncryptTdxLib.h>
  11. #include <Library/PcdLib.h>
  12. #include <Library/UefiBootServicesTableLib.h>
  13. #include "IoMmuInternal.h"
  14. extern BOOLEAN mReservedSharedMemSupported;
  15. #define SIZE_OF_MEM_RANGE(MemRange) (MemRange->HeaderSize + MemRange->DataSize)
  16. #define RESERVED_MEM_BITMAP_4K_MASK 0xf
  17. #define RESERVED_MEM_BITMAP_32K_MASK 0xff0
  18. #define RESERVED_MEM_BITMAP_128K_MASK 0x3000
  19. #define RESERVED_MEM_BITMAP_1M_MASK 0x40000
  20. #define RESERVED_MEM_BITMAP_2M_MASK 0x180000
  21. #define RESERVED_MEM_BITMAP_MASK 0x1fffff
  22. /**
  23. * mReservedMemRanges describes the layout of the reserved memory.
  24. * The reserved memory consists of disfferent size of memory region.
  25. * The pieces of memory with the same size are managed by one entry
  26. * in the mReservedMemRanges. All the pieces of memories are managed by
  27. * mReservedMemBitmap which is a UINT32. It means it can manage at most
  28. * 32 pieces of memory. Because of the layout of CommonBuffer
  29. * (1-page header + n-page data), a piece of reserved memory consists of
  30. * 2 parts: Header + Data.
  31. *
  32. * So put all these together, mReservedMemRanges and mReservedMemBitmap
  33. * are designed to manage the reserved memory.
  34. *
  35. * Use the second entry of mReservedMemRanges as an example.
  36. * { RESERVED_MEM_BITMAP_32K_MASK, 4, 8, SIZE_32KB, SIZE_4KB, 0 },
  37. * - RESERVED_MEM_BITMAP_32K_MASK is 0xff0. It means bit4-11 in mReservedMemBitmap
  38. * is reserved for 32K size memory.
  39. * - 4 is the shift of mReservedMemBitmap.
  40. * - 8 means there are 8 pieces of 32K size memory.
  41. * - SIZE_32KB indicates the size of Data part.
  42. * - SIZE_4KB is the size of Header part.
  43. * - 0 is the start address of this memory range which will be populated when
  44. * the reserved memory is initialized.
  45. *
  46. * The size and count of the memory region are derived from the experience. For
  47. * a typical grub boot, there are about 5100 IoMmu/DMA operation. Most of these
  48. * DMA operation require the memory with size less than 32K (~5080). But we find
  49. * in grub boot there may be 2 DMA operation which require for the memory larger
  50. * than 1M. And these 2 DMA operation occur concurrently. So we reserve 2 pieces
  51. * of memory with size of SIZE_2MB. This is for the best boot performance.
  52. *
  53. * If all the reserved memory are exausted, then it will fall back to the legacy
  54. * memory allocation as before.
  55. */
  56. STATIC IOMMU_RESERVED_MEM_RANGE mReservedMemRanges[] = {
  57. { RESERVED_MEM_BITMAP_4K_MASK, 0, 4, SIZE_4KB, SIZE_4KB, 0 },
  58. { RESERVED_MEM_BITMAP_32K_MASK, 4, 8, SIZE_32KB, SIZE_4KB, 0 },
  59. { RESERVED_MEM_BITMAP_128K_MASK, 12, 2, SIZE_128KB, SIZE_4KB, 0 },
  60. { RESERVED_MEM_BITMAP_1M_MASK, 14, 1, SIZE_1MB, SIZE_4KB, 0 },
  61. { RESERVED_MEM_BITMAP_2M_MASK, 15, 2, SIZE_2MB, SIZE_4KB, 0 },
  62. };
  63. //
  64. // Bitmap of the allocation of reserved memory.
  65. //
  66. STATIC UINT32 mReservedMemBitmap = 0;
  67. //
  68. // Start address of the reserved memory region.
  69. //
  70. STATIC EFI_PHYSICAL_ADDRESS mReservedSharedMemAddress = 0;
  71. //
  72. // Total size of the reserved memory region.
  73. //
  74. STATIC UINT32 mReservedSharedMemSize = 0;
  75. /**
  76. * Calculate the size of reserved memory.
  77. *
  78. * @retval UINT32 Size of the reserved memory
  79. */
  80. STATIC
  81. UINT32
  82. CalcuateReservedMemSize (
  83. VOID
  84. )
  85. {
  86. UINT32 Index;
  87. IOMMU_RESERVED_MEM_RANGE *MemRange;
  88. if (mReservedSharedMemSize != 0) {
  89. return mReservedSharedMemSize;
  90. }
  91. for (Index = 0; Index < ARRAY_SIZE (mReservedMemRanges); Index++) {
  92. MemRange = &mReservedMemRanges[Index];
  93. mReservedSharedMemSize += (SIZE_OF_MEM_RANGE (MemRange) * MemRange->Slots);
  94. }
  95. return mReservedSharedMemSize;
  96. }
  97. /**
  98. * Allocate a memory region and convert it to be shared. This memory region will be
  99. * used in the DMA operation.
  100. *
  101. * The pre-alloc memory contains pieces of memory regions with different size. The
  102. * allocation of the shared memory regions are indicated by a 32-bit bitmap (mReservedMemBitmap).
  103. *
  104. * The memory regions are consumed by IoMmuAllocateBuffer (in which CommonBuffer is allocated) and
  105. * IoMmuMap (in which bounce buffer is allocated).
  106. *
  107. * The CommonBuffer contains 2 parts, one page for CommonBufferHeader which is private memory,
  108. * the other part is shared memory. So the layout of a piece of memory region after initialization
  109. * looks like:
  110. *
  111. * |------------|----------------------------|
  112. * | Header | Data | <-- a piece of pre-alloc memory region
  113. * | 4k, private| 4k/32k/128k/etc, shared |
  114. * |-----------------------------------------|
  115. *
  116. * @retval EFI_SUCCESS Successfully initialize the reserved memory.
  117. * @retval EFI_UNSUPPORTED This feature is not supported.
  118. */
  119. EFI_STATUS
  120. IoMmuInitReservedSharedMem (
  121. VOID
  122. )
  123. {
  124. EFI_STATUS Status;
  125. UINT32 Index1, Index2;
  126. UINTN TotalPages;
  127. IOMMU_RESERVED_MEM_RANGE *MemRange;
  128. EFI_PHYSICAL_ADDRESS PhysicalAddress;
  129. UINT64 SharedAddress;
  130. if (!mReservedSharedMemSupported) {
  131. return EFI_UNSUPPORTED;
  132. }
  133. TotalPages = EFI_SIZE_TO_PAGES (CalcuateReservedMemSize ());
  134. PhysicalAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)AllocatePages (TotalPages);
  135. DEBUG ((
  136. DEBUG_VERBOSE,
  137. "%a: ReservedMem (%d pages) address = 0x%llx\n",
  138. __FUNCTION__,
  139. TotalPages,
  140. PhysicalAddress
  141. ));
  142. mReservedMemBitmap = 0;
  143. mReservedSharedMemAddress = PhysicalAddress;
  144. for (Index1 = 0; Index1 < ARRAY_SIZE (mReservedMemRanges); Index1++) {
  145. MemRange = &mReservedMemRanges[Index1];
  146. MemRange->StartAddressOfMemRange = PhysicalAddress;
  147. for (Index2 = 0; Index2 < MemRange->Slots; Index2++) {
  148. SharedAddress = (UINT64)(UINTN)(MemRange->StartAddressOfMemRange + Index2 * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize);
  149. if (CC_GUEST_IS_SEV (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
  150. Status = MemEncryptSevClearPageEncMask (
  151. 0,
  152. SharedAddress,
  153. EFI_SIZE_TO_PAGES (MemRange->DataSize)
  154. );
  155. ASSERT (!EFI_ERROR (Status));
  156. } else if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
  157. Status = MemEncryptTdxSetPageSharedBit (
  158. 0,
  159. SharedAddress,
  160. EFI_SIZE_TO_PAGES (MemRange->DataSize)
  161. );
  162. ASSERT (!EFI_ERROR (Status));
  163. } else {
  164. ASSERT (FALSE);
  165. }
  166. }
  167. PhysicalAddress += (MemRange->Slots * SIZE_OF_MEM_RANGE (MemRange));
  168. }
  169. return EFI_SUCCESS;
  170. }
  171. /**
  172. * Release the pre-alloc shared memory.
  173. *
  174. * @retval EFI_SUCCESS Successfully release the shared memory
  175. */
  176. EFI_STATUS
  177. IoMmuReleaseReservedSharedMem (
  178. BOOLEAN MemoryMapLocked
  179. )
  180. {
  181. EFI_STATUS Status;
  182. UINT32 Index1, Index2;
  183. IOMMU_RESERVED_MEM_RANGE *MemRange;
  184. UINT64 SharedAddress;
  185. if (!mReservedSharedMemSupported) {
  186. return EFI_SUCCESS;
  187. }
  188. for (Index1 = 0; Index1 < ARRAY_SIZE (mReservedMemRanges); Index1++) {
  189. MemRange = &mReservedMemRanges[Index1];
  190. for (Index2 = 0; Index2 < MemRange->Slots; Index2++) {
  191. SharedAddress = (UINT64)(UINTN)(MemRange->StartAddressOfMemRange + Index2 * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize);
  192. if (CC_GUEST_IS_SEV (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
  193. Status = MemEncryptSevSetPageEncMask (
  194. 0,
  195. SharedAddress,
  196. EFI_SIZE_TO_PAGES (MemRange->DataSize)
  197. );
  198. ASSERT (!EFI_ERROR (Status));
  199. } else if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
  200. Status = MemEncryptTdxClearPageSharedBit (
  201. 0,
  202. SharedAddress,
  203. EFI_SIZE_TO_PAGES (MemRange->DataSize)
  204. );
  205. ASSERT (!EFI_ERROR (Status));
  206. } else {
  207. ASSERT (FALSE);
  208. }
  209. }
  210. }
  211. if (!MemoryMapLocked) {
  212. FreePages ((VOID *)(UINTN)mReservedSharedMemAddress, EFI_SIZE_TO_PAGES (CalcuateReservedMemSize ()));
  213. mReservedSharedMemAddress = 0;
  214. mReservedMemBitmap = 0;
  215. }
  216. mReservedSharedMemSupported = FALSE;
  217. return EFI_SUCCESS;
  218. }
  219. /**
  220. * Allocate from the reserved memory pool.
  221. * If the reserved shared memory is exausted or there is no suitalbe size, it turns
  222. * to the LegacyAllocateBuffer.
  223. *
  224. * @param Type Allocate type
  225. * @param MemoryType The memory type to be allocated
  226. * @param Pages Pages to be allocated.
  227. * @param ReservedMemBitmap Bitmap of the allocated memory region
  228. * @param PhysicalAddress Pointer to the data part of allocated memory region
  229. *
  230. * @retval EFI_SUCCESS Successfully allocate the buffer
  231. * @retval Other As the error code indicates
  232. */
  233. STATIC
  234. EFI_STATUS
  235. InternalAllocateBuffer (
  236. IN EFI_ALLOCATE_TYPE Type,
  237. IN EFI_MEMORY_TYPE MemoryType,
  238. IN UINTN Pages,
  239. IN OUT UINT32 *ReservedMemBitmap,
  240. IN OUT EFI_PHYSICAL_ADDRESS *PhysicalAddress
  241. )
  242. {
  243. UINT32 MemBitmap;
  244. UINT8 Index;
  245. IOMMU_RESERVED_MEM_RANGE *MemRange;
  246. UINTN PagesOfLastMemRange;
  247. *ReservedMemBitmap = 0;
  248. if (Pages == 0) {
  249. ASSERT (FALSE);
  250. return EFI_INVALID_PARAMETER;
  251. }
  252. if (!mReservedSharedMemSupported) {
  253. goto LegacyAllocateBuffer;
  254. }
  255. if (mReservedSharedMemAddress == 0) {
  256. goto LegacyAllocateBuffer;
  257. }
  258. PagesOfLastMemRange = 0;
  259. for (Index = 0; Index < ARRAY_SIZE (mReservedMemRanges); Index++) {
  260. if ((Pages > PagesOfLastMemRange) && (Pages <= EFI_SIZE_TO_PAGES (mReservedMemRanges[Index].DataSize))) {
  261. break;
  262. }
  263. PagesOfLastMemRange = EFI_SIZE_TO_PAGES (mReservedMemRanges[Index].DataSize);
  264. }
  265. if (Index == ARRAY_SIZE (mReservedMemRanges)) {
  266. // There is no suitable size of reserved memory. Turn to legacy allocate.
  267. goto LegacyAllocateBuffer;
  268. }
  269. MemRange = &mReservedMemRanges[Index];
  270. if ((mReservedMemBitmap & MemRange->BitmapMask) == MemRange->BitmapMask) {
  271. // The reserved memory is exausted. Turn to legacy allocate.
  272. goto LegacyAllocateBuffer;
  273. }
  274. MemBitmap = (mReservedMemBitmap & MemRange->BitmapMask) >> MemRange->Shift;
  275. for (Index = 0; Index < MemRange->Slots; Index++) {
  276. if ((MemBitmap & (UINT8)(1<<Index)) == 0) {
  277. break;
  278. }
  279. }
  280. ASSERT (Index != MemRange->Slots);
  281. *PhysicalAddress = MemRange->StartAddressOfMemRange + Index * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize;
  282. *ReservedMemBitmap = (UINT32)(1 << (Index + MemRange->Shift));
  283. DEBUG ((
  284. DEBUG_VERBOSE,
  285. "%a: range-size: %lx, start-address=0x%llx, pages=0x%llx, bits=0x%lx, bitmap: %lx => %lx\n",
  286. __FUNCTION__,
  287. MemRange->DataSize,
  288. *PhysicalAddress,
  289. Pages,
  290. *ReservedMemBitmap,
  291. mReservedMemBitmap,
  292. mReservedMemBitmap | *ReservedMemBitmap
  293. ));
  294. return EFI_SUCCESS;
  295. LegacyAllocateBuffer:
  296. *ReservedMemBitmap = 0;
  297. return gBS->AllocatePages (Type, MemoryType, Pages, PhysicalAddress);
  298. }
  299. /**
  300. * Allocate reserved shared memory for bounce buffer.
  301. *
  302. * @param Type Allocate type
  303. * @param MemoryType The memory type to be allocated
  304. * @param MapInfo Pointer to the MAP_INFO
  305. *
  306. * @retval EFI_SUCCESS Successfully allocate the bounce buffer
  307. * @retval Other As the error code indicates
  308. */
  309. EFI_STATUS
  310. IoMmuAllocateBounceBuffer (
  311. IN EFI_ALLOCATE_TYPE Type,
  312. IN EFI_MEMORY_TYPE MemoryType,
  313. IN OUT MAP_INFO *MapInfo
  314. )
  315. {
  316. EFI_STATUS Status;
  317. UINT32 ReservedMemBitmap;
  318. ReservedMemBitmap = 0;
  319. Status = InternalAllocateBuffer (
  320. Type,
  321. MemoryType,
  322. MapInfo->NumberOfPages,
  323. &ReservedMemBitmap,
  324. &MapInfo->PlainTextAddress
  325. );
  326. MapInfo->ReservedMemBitmap = ReservedMemBitmap;
  327. mReservedMemBitmap |= ReservedMemBitmap;
  328. ASSERT (Status == EFI_SUCCESS);
  329. return Status;
  330. }
  331. /**
  332. * Free the bounce buffer allocated in IoMmuAllocateBounceBuffer.
  333. *
  334. * @param MapInfo Pointer to the MAP_INFO
  335. * @return EFI_SUCCESS Successfully free the bounce buffer.
  336. */
  337. EFI_STATUS
  338. IoMmuFreeBounceBuffer (
  339. IN OUT MAP_INFO *MapInfo
  340. )
  341. {
  342. if (MapInfo->ReservedMemBitmap == 0) {
  343. gBS->FreePages (MapInfo->PlainTextAddress, MapInfo->NumberOfPages);
  344. } else {
  345. DEBUG ((
  346. DEBUG_VERBOSE,
  347. "%a: PlainTextAddress=0x%Lx, bits=0x%Lx, bitmap: %Lx => %Lx\n",
  348. __FUNCTION__,
  349. MapInfo->PlainTextAddress,
  350. MapInfo->ReservedMemBitmap,
  351. mReservedMemBitmap,
  352. mReservedMemBitmap & ((UINT32)(~MapInfo->ReservedMemBitmap))
  353. ));
  354. MapInfo->PlainTextAddress = 0;
  355. mReservedMemBitmap &= (UINT32)(~MapInfo->ReservedMemBitmap);
  356. MapInfo->ReservedMemBitmap = 0;
  357. }
  358. return EFI_SUCCESS;
  359. }
  360. /**
  361. * Allocate CommonBuffer from pre-allocated shared memory.
  362. *
  363. * @param MemoryType Memory type
  364. * @param CommonBufferPages Pages of CommonBuffer
  365. * @param PhysicalAddress Allocated physical address
  366. * @param ReservedMemBitmap Bitmap which indicates the allocation of reserved memory
  367. *
  368. * @retval EFI_SUCCESS Successfully allocate the common buffer
  369. * @retval Other As the error code indicates
  370. */
  371. EFI_STATUS
  372. IoMmuAllocateCommonBuffer (
  373. IN EFI_MEMORY_TYPE MemoryType,
  374. IN UINTN CommonBufferPages,
  375. OUT EFI_PHYSICAL_ADDRESS *PhysicalAddress,
  376. OUT UINT32 *ReservedMemBitmap
  377. )
  378. {
  379. EFI_STATUS Status;
  380. Status = InternalAllocateBuffer (
  381. AllocateMaxAddress,
  382. MemoryType,
  383. CommonBufferPages,
  384. ReservedMemBitmap,
  385. PhysicalAddress
  386. );
  387. ASSERT (Status == EFI_SUCCESS);
  388. mReservedMemBitmap |= *ReservedMemBitmap;
  389. if (*ReservedMemBitmap != 0) {
  390. *PhysicalAddress -= SIZE_4KB;
  391. }
  392. return Status;
  393. }
  394. /**
  395. * Free CommonBuffer which is allocated by IoMmuAllocateCommonBuffer().
  396. *
  397. * @param CommonBufferHeader Pointer to the CommonBufferHeader
  398. * @param CommonBufferPages Pages of CommonBuffer
  399. *
  400. * @retval EFI_SUCCESS Successfully free the common buffer
  401. * @retval Other As the error code indicates
  402. */
  403. EFI_STATUS
  404. IoMmuFreeCommonBuffer (
  405. IN COMMON_BUFFER_HEADER *CommonBufferHeader,
  406. IN UINTN CommonBufferPages
  407. )
  408. {
  409. if (!mReservedSharedMemSupported) {
  410. goto LegacyFreeCommonBuffer;
  411. }
  412. if (CommonBufferHeader->ReservedMemBitmap == 0) {
  413. goto LegacyFreeCommonBuffer;
  414. }
  415. DEBUG ((
  416. DEBUG_VERBOSE,
  417. "%a: CommonBuffer=0x%Lx, bits=0x%Lx, bitmap: %Lx => %Lx\n",
  418. __FUNCTION__,
  419. (UINT64)(UINTN)CommonBufferHeader + SIZE_4KB,
  420. CommonBufferHeader->ReservedMemBitmap,
  421. mReservedMemBitmap,
  422. mReservedMemBitmap & ((UINT32)(~CommonBufferHeader->ReservedMemBitmap))
  423. ));
  424. mReservedMemBitmap &= (UINT32)(~CommonBufferHeader->ReservedMemBitmap);
  425. return EFI_SUCCESS;
  426. LegacyFreeCommonBuffer:
  427. return gBS->FreePages ((UINTN)CommonBufferHeader, CommonBufferPages);
  428. }