CpuPageTable.c 43 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439
  1. /** @file
  2. Page table management support.
  3. Copyright (c) 2017 - 2019, Intel Corporation. All rights reserved.<BR>
  4. Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
  5. SPDX-License-Identifier: BSD-2-Clause-Patent
  6. **/
  7. #include <Base.h>
  8. #include <Uefi.h>
  9. #include <Library/PeCoffGetEntryPointLib.h>
  10. #include <Library/SerialPortLib.h>
  11. #include <Library/SynchronizationLib.h>
  12. #include <Library/PrintLib.h>
  13. #include <Protocol/SmmBase2.h>
  14. #include <Register/Intel/Cpuid.h>
  15. #include <Register/Intel/Msr.h>
  16. #include "CpuDxe.h"
  17. #include "CpuPageTable.h"
  18. ///
  19. /// Page Table Entry
  20. ///
  21. #define IA32_PG_P BIT0
  22. #define IA32_PG_RW BIT1
  23. #define IA32_PG_U BIT2
  24. #define IA32_PG_WT BIT3
  25. #define IA32_PG_CD BIT4
  26. #define IA32_PG_A BIT5
  27. #define IA32_PG_D BIT6
  28. #define IA32_PG_PS BIT7
  29. #define IA32_PG_PAT_2M BIT12
  30. #define IA32_PG_PAT_4K IA32_PG_PS
  31. #define IA32_PG_PMNT BIT62
  32. #define IA32_PG_NX BIT63
  33. #define PAGE_ATTRIBUTE_BITS (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
  34. #define PAGE_ATTRIBUTE_BITS_POST_SPLIT (IA32_PG_RW | IA32_PG_P)
  35. //
  36. // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
  37. // X64 PAE PDPTE does not have such restriction
  38. //
  39. #define IA32_PAE_PDPTE_ATTRIBUTE_BITS (IA32_PG_P)
  40. #define PAGE_PROGATE_BITS (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
  41. #define PAGING_4K_MASK 0xFFF
  42. #define PAGING_2M_MASK 0x1FFFFF
  43. #define PAGING_1G_MASK 0x3FFFFFFF
  44. #define PAGING_PAE_INDEX_MASK 0x1FF
  45. #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
  46. #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
  47. #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
  48. #define MAX_PF_ENTRY_COUNT 10
  49. #define MAX_DEBUG_MESSAGE_LENGTH 0x100
  50. #define IA32_PF_EC_ID BIT4
  51. typedef enum {
  52. PageNone,
  53. Page4K,
  54. Page2M,
  55. Page1G,
  56. } PAGE_ATTRIBUTE;
  57. typedef struct {
  58. PAGE_ATTRIBUTE Attribute;
  59. UINT64 Length;
  60. UINT64 AddressMask;
  61. } PAGE_ATTRIBUTE_TABLE;
  62. typedef enum {
  63. PageActionAssign,
  64. PageActionSet,
  65. PageActionClear,
  66. } PAGE_ACTION;
  67. PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {
  68. { Page4K, SIZE_4KB, PAGING_4K_ADDRESS_MASK_64 },
  69. { Page2M, SIZE_2MB, PAGING_2M_ADDRESS_MASK_64 },
  70. { Page1G, SIZE_1GB, PAGING_1G_ADDRESS_MASK_64 },
  71. };
  72. PAGE_TABLE_POOL *mPageTablePool = NULL;
  73. BOOLEAN mPageTablePoolLock = FALSE;
  74. PAGE_TABLE_LIB_PAGING_CONTEXT mPagingContext;
  75. EFI_SMM_BASE2_PROTOCOL *mSmmBase2 = NULL;
  76. //
  77. // Record the page fault exception count for one instruction execution.
  78. //
  79. UINTN *mPFEntryCount;
  80. UINT64 *(*mLastPFEntryPointer)[MAX_PF_ENTRY_COUNT];
  81. /**
  82. Check if current execution environment is in SMM mode or not, via
  83. EFI_SMM_BASE2_PROTOCOL.
  84. This is necessary because of the fact that MdePkg\Library\SmmMemoryAllocationLib
  85. supports to free memory outside SMRAM. The library will call gBS->FreePool() or
  86. gBS->FreePages() and then SetMemorySpaceAttributes interface in turn to change
  87. memory paging attributes during free operation, if some memory related features
  88. are enabled (like Heap Guard).
  89. This means that SetMemorySpaceAttributes() has chance to run in SMM mode. This
  90. will cause incorrect result because SMM mode always loads its own page tables,
  91. which are usually different from DXE. This function can be used to detect such
  92. situation and help to avoid further misoperations.
  93. @retval TRUE In SMM mode.
  94. @retval FALSE Not in SMM mode.
  95. **/
  96. BOOLEAN
  97. IsInSmm (
  98. VOID
  99. )
  100. {
  101. BOOLEAN InSmm;
  102. InSmm = FALSE;
  103. if (mSmmBase2 == NULL) {
  104. gBS->LocateProtocol (&gEfiSmmBase2ProtocolGuid, NULL, (VOID **)&mSmmBase2);
  105. }
  106. if (mSmmBase2 != NULL) {
  107. mSmmBase2->InSmm (mSmmBase2, &InSmm);
  108. }
  109. //
  110. // mSmmBase2->InSmm() can only detect if the caller is running in SMRAM
  111. // or from SMM driver. It cannot tell if the caller is running in SMM mode.
  112. // Check page table base address to guarantee that because SMM mode willl
  113. // load its own page table.
  114. //
  115. return (InSmm &&
  116. mPagingContext.ContextData.X64.PageTableBase != (UINT64)AsmReadCr3 ());
  117. }
  118. /**
  119. Return current paging context.
  120. @param[in,out] PagingContext The paging context.
  121. **/
  122. VOID
  123. GetCurrentPagingContext (
  124. IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT *PagingContext
  125. )
  126. {
  127. UINT32 RegEax;
  128. CPUID_EXTENDED_CPU_SIG_EDX RegEdx;
  129. MSR_IA32_EFER_REGISTER MsrEfer;
  130. IA32_CR4 Cr4;
  131. IA32_CR0 Cr0;
  132. UINT32 *Attributes;
  133. UINTN *PageTableBase;
  134. //
  135. // Don't retrieve current paging context from processor if in SMM mode.
  136. //
  137. if (!IsInSmm ()) {
  138. ZeroMem (&mPagingContext, sizeof (mPagingContext));
  139. if (sizeof (UINTN) == sizeof (UINT64)) {
  140. mPagingContext.MachineType = IMAGE_FILE_MACHINE_X64;
  141. } else {
  142. mPagingContext.MachineType = IMAGE_FILE_MACHINE_I386;
  143. }
  144. GetPagingDetails (&mPagingContext.ContextData, &PageTableBase, &Attributes);
  145. Cr0.UintN = AsmReadCr0 ();
  146. Cr4.UintN = AsmReadCr4 ();
  147. if (Cr0.Bits.PG != 0) {
  148. *PageTableBase = (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64);
  149. } else {
  150. *PageTableBase = 0;
  151. }
  152. if (Cr0.Bits.WP != 0) {
  153. *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE;
  154. }
  155. if (Cr4.Bits.PSE != 0) {
  156. *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE;
  157. }
  158. if (Cr4.Bits.PAE != 0) {
  159. *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE;
  160. }
  161. if (Cr4.Bits.LA57 != 0) {
  162. *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_5_LEVEL;
  163. }
  164. AsmCpuid (CPUID_EXTENDED_FUNCTION, &RegEax, NULL, NULL, NULL);
  165. if (RegEax >= CPUID_EXTENDED_CPU_SIG) {
  166. AsmCpuid (CPUID_EXTENDED_CPU_SIG, NULL, NULL, NULL, &RegEdx.Uint32);
  167. if (RegEdx.Bits.NX != 0) {
  168. // XD supported
  169. MsrEfer.Uint64 = AsmReadMsr64 (MSR_CORE_IA32_EFER);
  170. if (MsrEfer.Bits.NXE != 0) {
  171. // XD activated
  172. *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED;
  173. }
  174. }
  175. if (RegEdx.Bits.Page1GB != 0) {
  176. *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT;
  177. }
  178. }
  179. }
  180. //
  181. // This can avoid getting SMM paging context if in SMM mode. We cannot assume
  182. // SMM mode shares the same paging context as DXE.
  183. //
  184. CopyMem (PagingContext, &mPagingContext, sizeof (mPagingContext));
  185. }
  186. /**
  187. Return length according to page attributes.
  188. @param[in] PageAttributes The page attribute of the page entry.
  189. @return The length of page entry.
  190. **/
  191. UINTN
  192. PageAttributeToLength (
  193. IN PAGE_ATTRIBUTE PageAttribute
  194. )
  195. {
  196. UINTN Index;
  197. for (Index = 0; Index < sizeof (mPageAttributeTable)/sizeof (mPageAttributeTable[0]); Index++) {
  198. if (PageAttribute == mPageAttributeTable[Index].Attribute) {
  199. return (UINTN)mPageAttributeTable[Index].Length;
  200. }
  201. }
  202. return 0;
  203. }
  204. /**
  205. Return address mask according to page attributes.
  206. @param[in] PageAttributes The page attribute of the page entry.
  207. @return The address mask of page entry.
  208. **/
  209. UINTN
  210. PageAttributeToMask (
  211. IN PAGE_ATTRIBUTE PageAttribute
  212. )
  213. {
  214. UINTN Index;
  215. for (Index = 0; Index < sizeof (mPageAttributeTable)/sizeof (mPageAttributeTable[0]); Index++) {
  216. if (PageAttribute == mPageAttributeTable[Index].Attribute) {
  217. return (UINTN)mPageAttributeTable[Index].AddressMask;
  218. }
  219. }
  220. return 0;
  221. }
  222. /**
  223. Return page table entry to match the address.
  224. @param[in] PagingContext The paging context.
  225. @param[in] Address The address to be checked.
  226. @param[out] PageAttributes The page attribute of the page entry.
  227. @return The page entry.
  228. **/
  229. VOID *
  230. GetPageTableEntry (
  231. IN PAGE_TABLE_LIB_PAGING_CONTEXT *PagingContext,
  232. IN PHYSICAL_ADDRESS Address,
  233. OUT PAGE_ATTRIBUTE *PageAttribute
  234. )
  235. {
  236. UINTN Index1;
  237. UINTN Index2;
  238. UINTN Index3;
  239. UINTN Index4;
  240. UINTN Index5;
  241. UINT64 *L1PageTable;
  242. UINT64 *L2PageTable;
  243. UINT64 *L3PageTable;
  244. UINT64 *L4PageTable;
  245. UINT64 *L5PageTable;
  246. UINT64 AddressEncMask;
  247. ASSERT (PagingContext != NULL);
  248. Index5 = ((UINTN)RShiftU64 (Address, 48)) & PAGING_PAE_INDEX_MASK;
  249. Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_PAE_INDEX_MASK;
  250. Index3 = ((UINTN)Address >> 30) & PAGING_PAE_INDEX_MASK;
  251. Index2 = ((UINTN)Address >> 21) & PAGING_PAE_INDEX_MASK;
  252. Index1 = ((UINTN)Address >> 12) & PAGING_PAE_INDEX_MASK;
  253. // Make sure AddressEncMask is contained to smallest supported address field.
  254. //
  255. AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
  256. if (AddressEncMask == 0) {
  257. AddressEncMask = PcdGet64 (PcdTdxSharedBitMask) & PAGING_1G_ADDRESS_MASK_64;
  258. }
  259. if (PagingContext->MachineType == IMAGE_FILE_MACHINE_X64) {
  260. if ((PagingContext->ContextData.X64.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_5_LEVEL) != 0) {
  261. L5PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
  262. if (L5PageTable[Index5] == 0) {
  263. *PageAttribute = PageNone;
  264. return NULL;
  265. }
  266. L4PageTable = (UINT64 *)(UINTN)(L5PageTable[Index5] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
  267. } else {
  268. L4PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
  269. }
  270. if (L4PageTable[Index4] == 0) {
  271. *PageAttribute = PageNone;
  272. return NULL;
  273. }
  274. L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
  275. } else {
  276. ASSERT ((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0);
  277. L3PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.Ia32.PageTableBase;
  278. }
  279. if (L3PageTable[Index3] == 0) {
  280. *PageAttribute = PageNone;
  281. return NULL;
  282. }
  283. if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {
  284. // 1G
  285. *PageAttribute = Page1G;
  286. return &L3PageTable[Index3];
  287. }
  288. L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
  289. if (L2PageTable[Index2] == 0) {
  290. *PageAttribute = PageNone;
  291. return NULL;
  292. }
  293. if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {
  294. // 2M
  295. *PageAttribute = Page2M;
  296. return &L2PageTable[Index2];
  297. }
  298. // 4k
  299. L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
  300. if ((L1PageTable[Index1] == 0) && (Address != 0)) {
  301. *PageAttribute = PageNone;
  302. return NULL;
  303. }
  304. *PageAttribute = Page4K;
  305. return &L1PageTable[Index1];
  306. }
  307. /**
  308. Return memory attributes of page entry.
  309. @param[in] PageEntry The page entry.
  310. @return Memory attributes of page entry.
  311. **/
  312. UINT64
  313. GetAttributesFromPageEntry (
  314. IN UINT64 *PageEntry
  315. )
  316. {
  317. UINT64 Attributes;
  318. Attributes = 0;
  319. if ((*PageEntry & IA32_PG_P) == 0) {
  320. Attributes |= EFI_MEMORY_RP;
  321. }
  322. if ((*PageEntry & IA32_PG_RW) == 0) {
  323. Attributes |= EFI_MEMORY_RO;
  324. }
  325. if ((*PageEntry & IA32_PG_NX) != 0) {
  326. Attributes |= EFI_MEMORY_XP;
  327. }
  328. return Attributes;
  329. }
  330. /**
  331. Modify memory attributes of page entry.
  332. @param[in] PagingContext The paging context.
  333. @param[in] PageEntry The page entry.
  334. @param[in] Attributes The bit mask of attributes to modify for the memory region.
  335. @param[in] PageAction The page action.
  336. @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
  337. **/
  338. VOID
  339. ConvertPageEntryAttribute (
  340. IN PAGE_TABLE_LIB_PAGING_CONTEXT *PagingContext,
  341. IN UINT64 *PageEntry,
  342. IN UINT64 Attributes,
  343. IN PAGE_ACTION PageAction,
  344. OUT BOOLEAN *IsModified
  345. )
  346. {
  347. UINT64 CurrentPageEntry;
  348. UINT64 NewPageEntry;
  349. UINT32 *PageAttributes;
  350. CurrentPageEntry = *PageEntry;
  351. NewPageEntry = CurrentPageEntry;
  352. if ((Attributes & EFI_MEMORY_RP) != 0) {
  353. switch (PageAction) {
  354. case PageActionAssign:
  355. case PageActionSet:
  356. NewPageEntry &= ~(UINT64)IA32_PG_P;
  357. break;
  358. case PageActionClear:
  359. NewPageEntry |= IA32_PG_P;
  360. break;
  361. }
  362. } else {
  363. switch (PageAction) {
  364. case PageActionAssign:
  365. NewPageEntry |= IA32_PG_P;
  366. break;
  367. case PageActionSet:
  368. case PageActionClear:
  369. break;
  370. }
  371. }
  372. if ((Attributes & EFI_MEMORY_RO) != 0) {
  373. switch (PageAction) {
  374. case PageActionAssign:
  375. case PageActionSet:
  376. NewPageEntry &= ~(UINT64)IA32_PG_RW;
  377. break;
  378. case PageActionClear:
  379. NewPageEntry |= IA32_PG_RW;
  380. break;
  381. }
  382. } else {
  383. switch (PageAction) {
  384. case PageActionAssign:
  385. NewPageEntry |= IA32_PG_RW;
  386. break;
  387. case PageActionSet:
  388. case PageActionClear:
  389. break;
  390. }
  391. }
  392. GetPagingDetails (&PagingContext->ContextData, NULL, &PageAttributes);
  393. if ((*PageAttributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED) != 0) {
  394. if ((Attributes & EFI_MEMORY_XP) != 0) {
  395. switch (PageAction) {
  396. case PageActionAssign:
  397. case PageActionSet:
  398. NewPageEntry |= IA32_PG_NX;
  399. break;
  400. case PageActionClear:
  401. NewPageEntry &= ~IA32_PG_NX;
  402. break;
  403. }
  404. } else {
  405. switch (PageAction) {
  406. case PageActionAssign:
  407. NewPageEntry &= ~IA32_PG_NX;
  408. break;
  409. case PageActionSet:
  410. case PageActionClear:
  411. break;
  412. }
  413. }
  414. }
  415. *PageEntry = NewPageEntry;
  416. if (CurrentPageEntry != NewPageEntry) {
  417. *IsModified = TRUE;
  418. DEBUG ((DEBUG_VERBOSE, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry));
  419. DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
  420. } else {
  421. *IsModified = FALSE;
  422. }
  423. }
  424. /**
  425. This function returns if there is need to split page entry.
  426. @param[in] BaseAddress The base address to be checked.
  427. @param[in] Length The length to be checked.
  428. @param[in] PageEntry The page entry to be checked.
  429. @param[in] PageAttribute The page attribute of the page entry.
  430. @retval SplitAttributes on if there is need to split page entry.
  431. **/
  432. PAGE_ATTRIBUTE
  433. NeedSplitPage (
  434. IN PHYSICAL_ADDRESS BaseAddress,
  435. IN UINT64 Length,
  436. IN UINT64 *PageEntry,
  437. IN PAGE_ATTRIBUTE PageAttribute
  438. )
  439. {
  440. UINT64 PageEntryLength;
  441. PageEntryLength = PageAttributeToLength (PageAttribute);
  442. if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
  443. return PageNone;
  444. }
  445. if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
  446. return Page4K;
  447. }
  448. return Page2M;
  449. }
  450. /**
  451. This function splits one page entry to small page entries.
  452. @param[in] PageEntry The page entry to be splitted.
  453. @param[in] PageAttribute The page attribute of the page entry.
  454. @param[in] SplitAttribute How to split the page entry.
  455. @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
  456. @retval RETURN_SUCCESS The page entry is splitted.
  457. @retval RETURN_UNSUPPORTED The page entry does not support to be splitted.
  458. @retval RETURN_OUT_OF_RESOURCES No resource to split page entry.
  459. **/
  460. RETURN_STATUS
  461. SplitPage (
  462. IN UINT64 *PageEntry,
  463. IN PAGE_ATTRIBUTE PageAttribute,
  464. IN PAGE_ATTRIBUTE SplitAttribute,
  465. IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc
  466. )
  467. {
  468. UINT64 BaseAddress;
  469. UINT64 *NewPageEntry;
  470. UINTN Index;
  471. UINT64 AddressEncMask;
  472. ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
  473. ASSERT (AllocatePagesFunc != NULL);
  474. // Make sure AddressEncMask is contained to smallest supported address field.
  475. //
  476. AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
  477. if (PageAttribute == Page2M) {
  478. //
  479. // Split 2M to 4K
  480. //
  481. ASSERT (SplitAttribute == Page4K);
  482. if (SplitAttribute == Page4K) {
  483. NewPageEntry = AllocatePagesFunc (1);
  484. DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
  485. if (NewPageEntry == NULL) {
  486. return RETURN_OUT_OF_RESOURCES;
  487. }
  488. BaseAddress = *PageEntry & ~AddressEncMask & PAGING_2M_ADDRESS_MASK_64;
  489. for (Index = 0; Index < SIZE_4KB / sizeof (UINT64); Index++) {
  490. NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);
  491. }
  492. (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | PAGE_ATTRIBUTE_BITS_POST_SPLIT;
  493. return RETURN_SUCCESS;
  494. } else {
  495. return RETURN_UNSUPPORTED;
  496. }
  497. } else if (PageAttribute == Page1G) {
  498. //
  499. // Split 1G to 2M
  500. // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
  501. //
  502. ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
  503. if (((SplitAttribute == Page2M) || (SplitAttribute == Page4K))) {
  504. NewPageEntry = AllocatePagesFunc (1);
  505. DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
  506. if (NewPageEntry == NULL) {
  507. return RETURN_OUT_OF_RESOURCES;
  508. }
  509. BaseAddress = *PageEntry & ~AddressEncMask & PAGING_1G_ADDRESS_MASK_64;
  510. for (Index = 0; Index < SIZE_4KB / sizeof (UINT64); Index++) {
  511. NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | AddressEncMask | IA32_PG_PS | ((*PageEntry) & PAGE_PROGATE_BITS);
  512. }
  513. (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | PAGE_ATTRIBUTE_BITS_POST_SPLIT;
  514. return RETURN_SUCCESS;
  515. } else {
  516. return RETURN_UNSUPPORTED;
  517. }
  518. } else {
  519. return RETURN_UNSUPPORTED;
  520. }
  521. }
  522. /**
  523. Check the WP status in CR0 register. This bit is used to lock or unlock write
  524. access to pages marked as read-only.
  525. @retval TRUE Write protection is enabled.
  526. @retval FALSE Write protection is disabled.
  527. **/
  528. BOOLEAN
  529. IsReadOnlyPageWriteProtected (
  530. VOID
  531. )
  532. {
  533. IA32_CR0 Cr0;
  534. //
  535. // To avoid unforseen consequences, don't touch paging settings in SMM mode
  536. // in this driver.
  537. //
  538. if (!IsInSmm ()) {
  539. Cr0.UintN = AsmReadCr0 ();
  540. return (BOOLEAN)(Cr0.Bits.WP != 0);
  541. }
  542. return FALSE;
  543. }
  544. /**
  545. Disable Write Protect on pages marked as read-only.
  546. **/
  547. VOID
  548. DisableReadOnlyPageWriteProtect (
  549. VOID
  550. )
  551. {
  552. IA32_CR0 Cr0;
  553. //
  554. // To avoid unforseen consequences, don't touch paging settings in SMM mode
  555. // in this driver.
  556. //
  557. if (!IsInSmm ()) {
  558. Cr0.UintN = AsmReadCr0 ();
  559. Cr0.Bits.WP = 0;
  560. AsmWriteCr0 (Cr0.UintN);
  561. }
  562. }
  563. /**
  564. Enable Write Protect on pages marked as read-only.
  565. **/
  566. VOID
  567. EnableReadOnlyPageWriteProtect (
  568. VOID
  569. )
  570. {
  571. IA32_CR0 Cr0;
  572. //
  573. // To avoid unforseen consequences, don't touch paging settings in SMM mode
  574. // in this driver.
  575. //
  576. if (!IsInSmm ()) {
  577. Cr0.UintN = AsmReadCr0 ();
  578. Cr0.Bits.WP = 1;
  579. AsmWriteCr0 (Cr0.UintN);
  580. }
  581. }
  582. /**
  583. This function modifies the page attributes for the memory region specified by BaseAddress and
  584. Length from their current attributes to the attributes specified by Attributes.
  585. Caller should make sure BaseAddress and Length is at page boundary.
  586. @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
  587. @param[in] BaseAddress The physical address that is the start address of a memory region.
  588. @param[in] Length The size in bytes of the memory region.
  589. @param[in] Attributes The bit mask of attributes to modify for the memory region.
  590. @param[in] PageAction The page action.
  591. @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
  592. NULL mean page split is unsupported.
  593. @param[out] IsSplitted TRUE means page table splitted. FALSE means page table not splitted.
  594. @param[out] IsModified TRUE means page table modified. FALSE means page table not modified.
  595. @retval RETURN_SUCCESS The attributes were modified for the memory region.
  596. @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
  597. BaseAddress and Length cannot be modified.
  598. @retval RETURN_INVALID_PARAMETER Length is zero.
  599. Attributes specified an illegal combination of attributes that
  600. cannot be set together.
  601. @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
  602. the memory resource range.
  603. @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
  604. resource range specified by BaseAddress and Length.
  605. The bit mask of attributes is not support for the memory resource
  606. range specified by BaseAddress and Length.
  607. **/
  608. RETURN_STATUS
  609. ConvertMemoryPageAttributes (
  610. IN PAGE_TABLE_LIB_PAGING_CONTEXT *PagingContext OPTIONAL,
  611. IN PHYSICAL_ADDRESS BaseAddress,
  612. IN UINT64 Length,
  613. IN UINT64 Attributes,
  614. IN PAGE_ACTION PageAction,
  615. IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL,
  616. OUT BOOLEAN *IsSplitted OPTIONAL,
  617. OUT BOOLEAN *IsModified OPTIONAL
  618. )
  619. {
  620. PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext;
  621. UINT64 *PageEntry;
  622. PAGE_ATTRIBUTE PageAttribute;
  623. UINTN PageEntryLength;
  624. PAGE_ATTRIBUTE SplitAttribute;
  625. RETURN_STATUS Status;
  626. BOOLEAN IsEntryModified;
  627. BOOLEAN IsWpEnabled;
  628. if ((BaseAddress & (SIZE_4KB - 1)) != 0) {
  629. DEBUG ((DEBUG_ERROR, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress));
  630. return EFI_UNSUPPORTED;
  631. }
  632. if ((Length & (SIZE_4KB - 1)) != 0) {
  633. DEBUG ((DEBUG_ERROR, "Length(0x%lx) is not aligned!\n", Length));
  634. return EFI_UNSUPPORTED;
  635. }
  636. if (Length == 0) {
  637. DEBUG ((DEBUG_ERROR, "Length is 0!\n"));
  638. return RETURN_INVALID_PARAMETER;
  639. }
  640. if ((Attributes & ~EFI_MEMORY_ATTRIBUTE_MASK) != 0) {
  641. DEBUG ((DEBUG_ERROR, "Attributes(0x%lx) has unsupported bit\n", Attributes));
  642. return EFI_UNSUPPORTED;
  643. }
  644. if (PagingContext == NULL) {
  645. GetCurrentPagingContext (&CurrentPagingContext);
  646. } else {
  647. CopyMem (&CurrentPagingContext, PagingContext, sizeof (CurrentPagingContext));
  648. }
  649. switch (CurrentPagingContext.MachineType) {
  650. case IMAGE_FILE_MACHINE_I386:
  651. if (CurrentPagingContext.ContextData.Ia32.PageTableBase == 0) {
  652. if (Attributes == 0) {
  653. return EFI_SUCCESS;
  654. } else {
  655. DEBUG ((DEBUG_ERROR, "PageTable is 0!\n"));
  656. return EFI_UNSUPPORTED;
  657. }
  658. }
  659. if ((CurrentPagingContext.ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) == 0) {
  660. DEBUG ((DEBUG_ERROR, "Non-PAE Paging!\n"));
  661. return EFI_UNSUPPORTED;
  662. }
  663. if ((BaseAddress + Length) > BASE_4GB) {
  664. DEBUG ((DEBUG_ERROR, "Beyond 4GB memory in 32-bit mode!\n"));
  665. return EFI_UNSUPPORTED;
  666. }
  667. break;
  668. case IMAGE_FILE_MACHINE_X64:
  669. ASSERT (CurrentPagingContext.ContextData.X64.PageTableBase != 0);
  670. break;
  671. default:
  672. ASSERT (FALSE);
  673. return EFI_UNSUPPORTED;
  674. break;
  675. }
  676. // DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
  677. if (IsSplitted != NULL) {
  678. *IsSplitted = FALSE;
  679. }
  680. if (IsModified != NULL) {
  681. *IsModified = FALSE;
  682. }
  683. if (AllocatePagesFunc == NULL) {
  684. AllocatePagesFunc = AllocatePageTableMemory;
  685. }
  686. //
  687. // Make sure that the page table is changeable.
  688. //
  689. IsWpEnabled = IsReadOnlyPageWriteProtected ();
  690. if (IsWpEnabled) {
  691. DisableReadOnlyPageWriteProtect ();
  692. }
  693. //
  694. // Below logic is to check 2M/4K page to make sure we do not waste memory.
  695. //
  696. Status = EFI_SUCCESS;
  697. while (Length != 0) {
  698. PageEntry = GetPageTableEntry (&CurrentPagingContext, BaseAddress, &PageAttribute);
  699. if (PageEntry == NULL) {
  700. Status = RETURN_UNSUPPORTED;
  701. goto Done;
  702. }
  703. PageEntryLength = PageAttributeToLength (PageAttribute);
  704. SplitAttribute = NeedSplitPage (BaseAddress, Length, PageEntry, PageAttribute);
  705. if (SplitAttribute == PageNone) {
  706. ConvertPageEntryAttribute (&CurrentPagingContext, PageEntry, Attributes, PageAction, &IsEntryModified);
  707. if (IsEntryModified) {
  708. if (IsModified != NULL) {
  709. *IsModified = TRUE;
  710. }
  711. }
  712. //
  713. // Convert success, move to next
  714. //
  715. BaseAddress += PageEntryLength;
  716. Length -= PageEntryLength;
  717. } else {
  718. if (AllocatePagesFunc == NULL) {
  719. Status = RETURN_UNSUPPORTED;
  720. goto Done;
  721. }
  722. Status = SplitPage (PageEntry, PageAttribute, SplitAttribute, AllocatePagesFunc);
  723. if (RETURN_ERROR (Status)) {
  724. Status = RETURN_UNSUPPORTED;
  725. goto Done;
  726. }
  727. if (IsSplitted != NULL) {
  728. *IsSplitted = TRUE;
  729. }
  730. if (IsModified != NULL) {
  731. *IsModified = TRUE;
  732. }
  733. //
  734. // Just split current page
  735. // Convert success in next around
  736. //
  737. }
  738. }
  739. Done:
  740. //
  741. // Restore page table write protection, if any.
  742. //
  743. if (IsWpEnabled) {
  744. EnableReadOnlyPageWriteProtect ();
  745. }
  746. return Status;
  747. }
  748. /**
  749. This function assigns the page attributes for the memory region specified by BaseAddress and
  750. Length from their current attributes to the attributes specified by Attributes.
  751. Caller should make sure BaseAddress and Length is at page boundary.
  752. Caller need guarantee the TPL <= TPL_NOTIFY, if there is split page request.
  753. @param[in] PagingContext The paging context. NULL means get page table from current CPU context.
  754. @param[in] BaseAddress The physical address that is the start address of a memory region.
  755. @param[in] Length The size in bytes of the memory region.
  756. @param[in] Attributes The bit mask of attributes to set for the memory region.
  757. @param[in] AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
  758. NULL mean page split is unsupported.
  759. @retval RETURN_SUCCESS The attributes were cleared for the memory region.
  760. @retval RETURN_ACCESS_DENIED The attributes for the memory resource range specified by
  761. BaseAddress and Length cannot be modified.
  762. @retval RETURN_INVALID_PARAMETER Length is zero.
  763. Attributes specified an illegal combination of attributes that
  764. cannot be set together.
  765. @retval RETURN_OUT_OF_RESOURCES There are not enough system resources to modify the attributes of
  766. the memory resource range.
  767. @retval RETURN_UNSUPPORTED The processor does not support one or more bytes of the memory
  768. resource range specified by BaseAddress and Length.
  769. The bit mask of attributes is not support for the memory resource
  770. range specified by BaseAddress and Length.
  771. **/
  772. RETURN_STATUS
  773. EFIAPI
  774. AssignMemoryPageAttributes (
  775. IN PAGE_TABLE_LIB_PAGING_CONTEXT *PagingContext OPTIONAL,
  776. IN PHYSICAL_ADDRESS BaseAddress,
  777. IN UINT64 Length,
  778. IN UINT64 Attributes,
  779. IN PAGE_TABLE_LIB_ALLOCATE_PAGES AllocatePagesFunc OPTIONAL
  780. )
  781. {
  782. RETURN_STATUS Status;
  783. BOOLEAN IsModified;
  784. BOOLEAN IsSplitted;
  785. // DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
  786. Status = ConvertMemoryPageAttributes (PagingContext, BaseAddress, Length, Attributes, PageActionAssign, AllocatePagesFunc, &IsSplitted, &IsModified);
  787. if (!EFI_ERROR (Status)) {
  788. if ((PagingContext == NULL) && IsModified) {
  789. //
  790. // Flush TLB as last step.
  791. //
  792. // Note: Since APs will always init CR3 register in HLT loop mode or do
  793. // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
  794. // here.
  795. //
  796. CpuFlushTlb ();
  797. }
  798. }
  799. return Status;
  800. }
  801. /**
  802. Check if Execute Disable feature is enabled or not.
  803. **/
  804. BOOLEAN
  805. IsExecuteDisableEnabled (
  806. VOID
  807. )
  808. {
  809. MSR_CORE_IA32_EFER_REGISTER MsrEfer;
  810. MsrEfer.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);
  811. return (MsrEfer.Bits.NXE == 1);
  812. }
  813. /**
  814. Update GCD memory space attributes according to current page table setup.
  815. **/
  816. VOID
  817. RefreshGcdMemoryAttributesFromPaging (
  818. VOID
  819. )
  820. {
  821. EFI_STATUS Status;
  822. UINTN NumberOfDescriptors;
  823. EFI_GCD_MEMORY_SPACE_DESCRIPTOR *MemorySpaceMap;
  824. PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext;
  825. PAGE_ATTRIBUTE PageAttribute;
  826. UINT64 *PageEntry;
  827. UINT64 PageLength;
  828. UINT64 MemorySpaceLength;
  829. UINT64 Length;
  830. UINT64 BaseAddress;
  831. UINT64 PageStartAddress;
  832. UINT64 Attributes;
  833. UINT64 Capabilities;
  834. UINT64 NewAttributes;
  835. UINTN Index;
  836. //
  837. // Assuming that memory space map returned is sorted already; otherwise sort
  838. // them in the order of lowest address to highest address.
  839. //
  840. Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
  841. ASSERT_EFI_ERROR (Status);
  842. GetCurrentPagingContext (&PagingContext);
  843. Attributes = 0;
  844. NewAttributes = 0;
  845. BaseAddress = 0;
  846. PageLength = 0;
  847. if (IsExecuteDisableEnabled ()) {
  848. Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP | EFI_MEMORY_XP;
  849. } else {
  850. Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP;
  851. }
  852. for (Index = 0; Index < NumberOfDescriptors; Index++) {
  853. if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
  854. continue;
  855. }
  856. //
  857. // Sync the actual paging related capabilities back to GCD service first.
  858. // As a side effect (good one), this can also help to avoid unnecessary
  859. // memory map entries due to the different capabilities of the same type
  860. // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
  861. // which could cause boot failure of some old Linux distro (before v4.3).
  862. //
  863. Status = gDS->SetMemorySpaceCapabilities (
  864. MemorySpaceMap[Index].BaseAddress,
  865. MemorySpaceMap[Index].Length,
  866. MemorySpaceMap[Index].Capabilities | Capabilities
  867. );
  868. if (EFI_ERROR (Status)) {
  869. //
  870. // If we cannot update the capabilities, we cannot update its
  871. // attributes either. So just simply skip current block of memory.
  872. //
  873. DEBUG ((
  874. DEBUG_WARN,
  875. "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
  876. (UINT64)Index,
  877. MemorySpaceMap[Index].BaseAddress,
  878. MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length - 1,
  879. MemorySpaceMap[Index].Capabilities,
  880. MemorySpaceMap[Index].Capabilities | Capabilities
  881. ));
  882. continue;
  883. }
  884. if (MemorySpaceMap[Index].BaseAddress >= (BaseAddress + PageLength)) {
  885. //
  886. // Current memory space starts at a new page. Resetting PageLength will
  887. // trigger a retrieval of page attributes at new address.
  888. //
  889. PageLength = 0;
  890. } else {
  891. //
  892. // In case current memory space is not adjacent to last one
  893. //
  894. PageLength -= (MemorySpaceMap[Index].BaseAddress - BaseAddress);
  895. }
  896. //
  897. // Sync actual page attributes to GCD
  898. //
  899. BaseAddress = MemorySpaceMap[Index].BaseAddress;
  900. MemorySpaceLength = MemorySpaceMap[Index].Length;
  901. while (MemorySpaceLength > 0) {
  902. if (PageLength == 0) {
  903. PageEntry = GetPageTableEntry (&PagingContext, BaseAddress, &PageAttribute);
  904. if (PageEntry == NULL) {
  905. break;
  906. }
  907. //
  908. // Note current memory space might start in the middle of a page
  909. //
  910. PageStartAddress = (*PageEntry) & (UINT64)PageAttributeToMask (PageAttribute);
  911. PageLength = PageAttributeToLength (PageAttribute) - (BaseAddress - PageStartAddress);
  912. Attributes = GetAttributesFromPageEntry (PageEntry);
  913. }
  914. Length = MIN (PageLength, MemorySpaceLength);
  915. if (Attributes != (MemorySpaceMap[Index].Attributes &
  916. EFI_MEMORY_ATTRIBUTE_MASK))
  917. {
  918. NewAttributes = (MemorySpaceMap[Index].Attributes &
  919. ~EFI_MEMORY_ATTRIBUTE_MASK) | Attributes;
  920. Status = gDS->SetMemorySpaceAttributes (
  921. BaseAddress,
  922. Length,
  923. NewAttributes
  924. );
  925. ASSERT_EFI_ERROR (Status);
  926. DEBUG ((
  927. DEBUG_VERBOSE,
  928. "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
  929. (UINT64)Index,
  930. BaseAddress,
  931. BaseAddress + Length - 1,
  932. MemorySpaceMap[Index].Attributes,
  933. NewAttributes
  934. ));
  935. }
  936. PageLength -= Length;
  937. MemorySpaceLength -= Length;
  938. BaseAddress += Length;
  939. }
  940. }
  941. FreePool (MemorySpaceMap);
  942. }
  943. /**
  944. Initialize a buffer pool for page table use only.
  945. To reduce the potential split operation on page table, the pages reserved for
  946. page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
  947. at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
  948. initialized with number of pages greater than or equal to the given PoolPages.
  949. Once the pages in the pool are used up, this method should be called again to
  950. reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
  951. often in practice.
  952. @param[in] PoolPages The least page number of the pool to be created.
  953. @retval TRUE The pool is initialized successfully.
  954. @retval FALSE The memory is out of resource.
  955. **/
  956. BOOLEAN
  957. InitializePageTablePool (
  958. IN UINTN PoolPages
  959. )
  960. {
  961. VOID *Buffer;
  962. BOOLEAN IsModified;
  963. //
  964. // Do not allow re-entrance.
  965. //
  966. if (mPageTablePoolLock) {
  967. return FALSE;
  968. }
  969. mPageTablePoolLock = TRUE;
  970. IsModified = FALSE;
  971. //
  972. // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
  973. // header.
  974. //
  975. PoolPages += 1; // Add one page for header.
  976. PoolPages = ((PoolPages - 1) / PAGE_TABLE_POOL_UNIT_PAGES + 1) *
  977. PAGE_TABLE_POOL_UNIT_PAGES;
  978. Buffer = AllocateAlignedPages (PoolPages, PAGE_TABLE_POOL_ALIGNMENT);
  979. if (Buffer == NULL) {
  980. DEBUG ((DEBUG_ERROR, "ERROR: Out of aligned pages\r\n"));
  981. goto Done;
  982. }
  983. DEBUG ((
  984. DEBUG_INFO,
  985. "Paging: added %lu pages to page table pool\r\n",
  986. (UINT64)PoolPages
  987. ));
  988. //
  989. // Link all pools into a list for easier track later.
  990. //
  991. if (mPageTablePool == NULL) {
  992. mPageTablePool = Buffer;
  993. mPageTablePool->NextPool = mPageTablePool;
  994. } else {
  995. ((PAGE_TABLE_POOL *)Buffer)->NextPool = mPageTablePool->NextPool;
  996. mPageTablePool->NextPool = Buffer;
  997. mPageTablePool = Buffer;
  998. }
  999. //
  1000. // Reserve one page for pool header.
  1001. //
  1002. mPageTablePool->FreePages = PoolPages - 1;
  1003. mPageTablePool->Offset = EFI_PAGES_TO_SIZE (1);
  1004. //
  1005. // Mark the whole pool pages as read-only.
  1006. //
  1007. ConvertMemoryPageAttributes (
  1008. NULL,
  1009. (PHYSICAL_ADDRESS)(UINTN)Buffer,
  1010. EFI_PAGES_TO_SIZE (PoolPages),
  1011. EFI_MEMORY_RO,
  1012. PageActionSet,
  1013. AllocatePageTableMemory,
  1014. NULL,
  1015. &IsModified
  1016. );
  1017. ASSERT (IsModified == TRUE);
  1018. Done:
  1019. mPageTablePoolLock = FALSE;
  1020. return IsModified;
  1021. }
  1022. /**
  1023. This API provides a way to allocate memory for page table.
  1024. This API can be called more than once to allocate memory for page tables.
  1025. Allocates the number of 4KB pages and returns a pointer to the allocated
  1026. buffer. The buffer returned is aligned on a 4KB boundary.
  1027. If Pages is 0, then NULL is returned.
  1028. If there is not enough memory remaining to satisfy the request, then NULL is
  1029. returned.
  1030. @param Pages The number of 4 KB pages to allocate.
  1031. @return A pointer to the allocated buffer or NULL if allocation fails.
  1032. **/
  1033. VOID *
  1034. EFIAPI
  1035. AllocatePageTableMemory (
  1036. IN UINTN Pages
  1037. )
  1038. {
  1039. VOID *Buffer;
  1040. if (Pages == 0) {
  1041. return NULL;
  1042. }
  1043. //
  1044. // Renew the pool if necessary.
  1045. //
  1046. if ((mPageTablePool == NULL) ||
  1047. (Pages > mPageTablePool->FreePages))
  1048. {
  1049. if (!InitializePageTablePool (Pages)) {
  1050. return NULL;
  1051. }
  1052. }
  1053. Buffer = (UINT8 *)mPageTablePool + mPageTablePool->Offset;
  1054. mPageTablePool->Offset += EFI_PAGES_TO_SIZE (Pages);
  1055. mPageTablePool->FreePages -= Pages;
  1056. return Buffer;
  1057. }
  1058. /**
  1059. Special handler for #DB exception, which will restore the page attributes
  1060. (not-present). It should work with #PF handler which will set pages to
  1061. 'present'.
  1062. @param ExceptionType Exception type.
  1063. @param SystemContext Pointer to EFI_SYSTEM_CONTEXT.
  1064. **/
  1065. VOID
  1066. EFIAPI
  1067. DebugExceptionHandler (
  1068. IN EFI_EXCEPTION_TYPE ExceptionType,
  1069. IN EFI_SYSTEM_CONTEXT SystemContext
  1070. )
  1071. {
  1072. UINTN CpuIndex;
  1073. UINTN PFEntry;
  1074. BOOLEAN IsWpEnabled;
  1075. MpInitLibWhoAmI (&CpuIndex);
  1076. //
  1077. // Clear last PF entries
  1078. //
  1079. IsWpEnabled = IsReadOnlyPageWriteProtected ();
  1080. if (IsWpEnabled) {
  1081. DisableReadOnlyPageWriteProtect ();
  1082. }
  1083. for (PFEntry = 0; PFEntry < mPFEntryCount[CpuIndex]; PFEntry++) {
  1084. if (mLastPFEntryPointer[CpuIndex][PFEntry] != NULL) {
  1085. *mLastPFEntryPointer[CpuIndex][PFEntry] &= ~(UINT64)IA32_PG_P;
  1086. }
  1087. }
  1088. if (IsWpEnabled) {
  1089. EnableReadOnlyPageWriteProtect ();
  1090. }
  1091. //
  1092. // Reset page fault exception count for next page fault.
  1093. //
  1094. mPFEntryCount[CpuIndex] = 0;
  1095. //
  1096. // Flush TLB
  1097. //
  1098. CpuFlushTlb ();
  1099. //
  1100. // Clear TF in EFLAGS
  1101. //
  1102. if (mPagingContext.MachineType == IMAGE_FILE_MACHINE_I386) {
  1103. SystemContext.SystemContextIa32->Eflags &= (UINT32) ~BIT8;
  1104. } else {
  1105. SystemContext.SystemContextX64->Rflags &= (UINT64) ~BIT8;
  1106. }
  1107. }
  1108. /**
  1109. Special handler for #PF exception, which will set the pages which caused
  1110. #PF to be 'present'. The attribute of those pages should be restored in
  1111. the subsequent #DB handler.
  1112. @param ExceptionType Exception type.
  1113. @param SystemContext Pointer to EFI_SYSTEM_CONTEXT.
  1114. **/
  1115. VOID
  1116. EFIAPI
  1117. PageFaultExceptionHandler (
  1118. IN EFI_EXCEPTION_TYPE ExceptionType,
  1119. IN EFI_SYSTEM_CONTEXT SystemContext
  1120. )
  1121. {
  1122. EFI_STATUS Status;
  1123. UINT64 PFAddress;
  1124. PAGE_TABLE_LIB_PAGING_CONTEXT PagingContext;
  1125. PAGE_ATTRIBUTE PageAttribute;
  1126. UINT64 Attributes;
  1127. UINT64 *PageEntry;
  1128. UINTN Index;
  1129. UINTN CpuIndex;
  1130. UINTN PageNumber;
  1131. BOOLEAN NonStopMode;
  1132. PFAddress = AsmReadCr2 () & ~EFI_PAGE_MASK;
  1133. if (PFAddress < BASE_4KB) {
  1134. NonStopMode = NULL_DETECTION_NONSTOP_MODE ? TRUE : FALSE;
  1135. } else {
  1136. NonStopMode = HEAP_GUARD_NONSTOP_MODE ? TRUE : FALSE;
  1137. }
  1138. if (NonStopMode) {
  1139. MpInitLibWhoAmI (&CpuIndex);
  1140. GetCurrentPagingContext (&PagingContext);
  1141. //
  1142. // Memory operation cross page boundary, like "rep mov" instruction, will
  1143. // cause infinite loop between this and Debug Trap handler. We have to make
  1144. // sure that current page and the page followed are both in PRESENT state.
  1145. //
  1146. PageNumber = 2;
  1147. while (PageNumber > 0) {
  1148. PageEntry = GetPageTableEntry (&PagingContext, PFAddress, &PageAttribute);
  1149. ASSERT (PageEntry != NULL);
  1150. if (PageEntry != NULL) {
  1151. Attributes = GetAttributesFromPageEntry (PageEntry);
  1152. if ((Attributes & EFI_MEMORY_RP) != 0) {
  1153. Attributes &= ~EFI_MEMORY_RP;
  1154. Status = AssignMemoryPageAttributes (
  1155. &PagingContext,
  1156. PFAddress,
  1157. EFI_PAGE_SIZE,
  1158. Attributes,
  1159. NULL
  1160. );
  1161. if (!EFI_ERROR (Status)) {
  1162. Index = mPFEntryCount[CpuIndex];
  1163. //
  1164. // Re-retrieve page entry because above calling might update page
  1165. // table due to table split.
  1166. //
  1167. PageEntry = GetPageTableEntry (&PagingContext, PFAddress, &PageAttribute);
  1168. mLastPFEntryPointer[CpuIndex][Index++] = PageEntry;
  1169. mPFEntryCount[CpuIndex] = Index;
  1170. }
  1171. }
  1172. }
  1173. PFAddress += EFI_PAGE_SIZE;
  1174. --PageNumber;
  1175. }
  1176. }
  1177. //
  1178. // Initialize the serial port before dumping.
  1179. //
  1180. SerialPortInitialize ();
  1181. //
  1182. // Display ExceptionType, CPU information and Image information
  1183. //
  1184. DumpCpuContext (ExceptionType, SystemContext);
  1185. if (NonStopMode) {
  1186. //
  1187. // Set TF in EFLAGS
  1188. //
  1189. if (mPagingContext.MachineType == IMAGE_FILE_MACHINE_I386) {
  1190. SystemContext.SystemContextIa32->Eflags |= (UINT32)BIT8;
  1191. } else {
  1192. SystemContext.SystemContextX64->Rflags |= (UINT64)BIT8;
  1193. }
  1194. } else {
  1195. CpuDeadLoop ();
  1196. }
  1197. }
  1198. /**
  1199. Initialize the Page Table lib.
  1200. **/
  1201. VOID
  1202. InitializePageTableLib (
  1203. VOID
  1204. )
  1205. {
  1206. PAGE_TABLE_LIB_PAGING_CONTEXT CurrentPagingContext;
  1207. UINT32 *Attributes;
  1208. UINTN *PageTableBase;
  1209. GetCurrentPagingContext (&CurrentPagingContext);
  1210. GetPagingDetails (&CurrentPagingContext.ContextData, &PageTableBase, &Attributes);
  1211. //
  1212. // Reserve memory of page tables for future uses, if paging is enabled.
  1213. //
  1214. if ((*PageTableBase != 0) &&
  1215. ((*Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0))
  1216. {
  1217. DisableReadOnlyPageWriteProtect ();
  1218. InitializePageTablePool (1);
  1219. EnableReadOnlyPageWriteProtect ();
  1220. }
  1221. if (HEAP_GUARD_NONSTOP_MODE || NULL_DETECTION_NONSTOP_MODE) {
  1222. mPFEntryCount = (UINTN *)AllocateZeroPool (sizeof (UINTN) * mNumberOfProcessors);
  1223. ASSERT (mPFEntryCount != NULL);
  1224. mLastPFEntryPointer = (UINT64 *(*)[MAX_PF_ENTRY_COUNT])
  1225. AllocateZeroPool (sizeof (mLastPFEntryPointer[0]) * mNumberOfProcessors);
  1226. ASSERT (mLastPFEntryPointer != NULL);
  1227. }
  1228. DEBUG ((DEBUG_INFO, "CurrentPagingContext:\n"));
  1229. DEBUG ((DEBUG_INFO, " MachineType - 0x%x\n", CurrentPagingContext.MachineType));
  1230. DEBUG ((DEBUG_INFO, " PageTableBase - 0x%Lx\n", (UINT64)*PageTableBase));
  1231. DEBUG ((DEBUG_INFO, " Attributes - 0x%x\n", *Attributes));
  1232. return;
  1233. }