MpService.c 66 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210
  1. /** @file
  2. SMM MP service implementation
  3. Copyright (c) 2009 - 2022, Intel Corporation. All rights reserved.<BR>
  4. Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
  5. SPDX-License-Identifier: BSD-2-Clause-Patent
  6. **/
  7. #include "PiSmmCpuDxeSmm.h"
  8. //
  9. // Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
  10. //
  11. MTRR_SETTINGS gSmiMtrrs;
  12. UINT64 gPhyMask;
  13. SMM_DISPATCHER_MP_SYNC_DATA *mSmmMpSyncData = NULL;
  14. UINTN mSmmMpSyncDataSize;
  15. SMM_CPU_SEMAPHORES mSmmCpuSemaphores;
  16. UINTN mSemaphoreSize;
  17. SPIN_LOCK *mPFLock = NULL;
  18. SMM_CPU_SYNC_MODE mCpuSmmSyncMode;
  19. BOOLEAN mMachineCheckSupported = FALSE;
  20. MM_COMPLETION mSmmStartupThisApToken;
  21. //
  22. // Processor specified by mPackageFirstThreadIndex[PackageIndex] will do the package-scope register check.
  23. //
  24. UINT32 *mPackageFirstThreadIndex = NULL;
  25. extern UINTN mSmmShadowStackSize;
  26. /**
  27. Performs an atomic compare exchange operation to get semaphore.
  28. The compare exchange operation must be performed using
  29. MP safe mechanisms.
  30. @param Sem IN: 32-bit unsigned integer
  31. OUT: original integer - 1
  32. @return Original integer - 1
  33. **/
  34. UINT32
  35. WaitForSemaphore (
  36. IN OUT volatile UINT32 *Sem
  37. )
  38. {
  39. UINT32 Value;
  40. for ( ; ;) {
  41. Value = *Sem;
  42. if ((Value != 0) &&
  43. (InterlockedCompareExchange32 (
  44. (UINT32 *)Sem,
  45. Value,
  46. Value - 1
  47. ) == Value))
  48. {
  49. break;
  50. }
  51. CpuPause ();
  52. }
  53. return Value - 1;
  54. }
  55. /**
  56. Performs an atomic compare exchange operation to release semaphore.
  57. The compare exchange operation must be performed using
  58. MP safe mechanisms.
  59. @param Sem IN: 32-bit unsigned integer
  60. OUT: original integer + 1
  61. @return Original integer + 1
  62. **/
  63. UINT32
  64. ReleaseSemaphore (
  65. IN OUT volatile UINT32 *Sem
  66. )
  67. {
  68. UINT32 Value;
  69. do {
  70. Value = *Sem;
  71. } while (Value + 1 != 0 &&
  72. InterlockedCompareExchange32 (
  73. (UINT32 *)Sem,
  74. Value,
  75. Value + 1
  76. ) != Value);
  77. return Value + 1;
  78. }
  79. /**
  80. Performs an atomic compare exchange operation to lock semaphore.
  81. The compare exchange operation must be performed using
  82. MP safe mechanisms.
  83. @param Sem IN: 32-bit unsigned integer
  84. OUT: -1
  85. @return Original integer
  86. **/
  87. UINT32
  88. LockdownSemaphore (
  89. IN OUT volatile UINT32 *Sem
  90. )
  91. {
  92. UINT32 Value;
  93. do {
  94. Value = *Sem;
  95. } while (InterlockedCompareExchange32 (
  96. (UINT32 *)Sem,
  97. Value,
  98. (UINT32)-1
  99. ) != Value);
  100. return Value;
  101. }
  102. /**
  103. Wait all APs to performs an atomic compare exchange operation to release semaphore.
  104. @param NumberOfAPs AP number
  105. **/
  106. VOID
  107. WaitForAllAPs (
  108. IN UINTN NumberOfAPs
  109. )
  110. {
  111. UINTN BspIndex;
  112. BspIndex = mSmmMpSyncData->BspIndex;
  113. while (NumberOfAPs-- > 0) {
  114. WaitForSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
  115. }
  116. }
  117. /**
  118. Performs an atomic compare exchange operation to release semaphore
  119. for each AP.
  120. **/
  121. VOID
  122. ReleaseAllAPs (
  123. VOID
  124. )
  125. {
  126. UINTN Index;
  127. for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
  128. if (IsPresentAp (Index)) {
  129. ReleaseSemaphore (mSmmMpSyncData->CpuData[Index].Run);
  130. }
  131. }
  132. }
  133. /**
  134. Check whether the index of CPU perform the package level register
  135. programming during System Management Mode initialization.
  136. The index of Processor specified by mPackageFirstThreadIndex[PackageIndex]
  137. will do the package-scope register programming.
  138. @param[in] CpuIndex Processor Index.
  139. @retval TRUE Perform the package level register programming.
  140. @retval FALSE Don't perform the package level register programming.
  141. **/
  142. BOOLEAN
  143. IsPackageFirstThread (
  144. IN UINTN CpuIndex
  145. )
  146. {
  147. UINT32 PackageIndex;
  148. PackageIndex = gSmmCpuPrivate->ProcessorInfo[CpuIndex].Location.Package;
  149. ASSERT (mPackageFirstThreadIndex != NULL);
  150. //
  151. // Set the value of mPackageFirstThreadIndex[PackageIndex].
  152. // The package-scope register are checked by the first processor (CpuIndex) in Package.
  153. //
  154. // If mPackageFirstThreadIndex[PackageIndex] equals to (UINT32)-1, then update
  155. // to current CpuIndex. If it doesn't equal to (UINT32)-1, don't change it.
  156. //
  157. if (mPackageFirstThreadIndex[PackageIndex] == (UINT32)-1) {
  158. mPackageFirstThreadIndex[PackageIndex] = (UINT32)CpuIndex;
  159. }
  160. return (BOOLEAN)(mPackageFirstThreadIndex[PackageIndex] == CpuIndex);
  161. }
  162. /**
  163. Returns the Number of SMM Delayed & Blocked & Disabled Thread Count.
  164. @param[in,out] DelayedCount The Number of SMM Delayed Thread Count.
  165. @param[in,out] BlockedCount The Number of SMM Blocked Thread Count.
  166. @param[in,out] DisabledCount The Number of SMM Disabled Thread Count.
  167. **/
  168. VOID
  169. GetSmmDelayedBlockedDisabledCount (
  170. IN OUT UINT32 *DelayedCount,
  171. IN OUT UINT32 *BlockedCount,
  172. IN OUT UINT32 *DisabledCount
  173. )
  174. {
  175. UINTN Index;
  176. for (Index = 0; Index < mNumberOfCpus; Index++) {
  177. if (IsPackageFirstThread (Index)) {
  178. if (DelayedCount != NULL) {
  179. *DelayedCount += (UINT32)SmmCpuFeaturesGetSmmRegister (Index, SmmRegSmmDelayed);
  180. }
  181. if (BlockedCount != NULL) {
  182. *BlockedCount += (UINT32)SmmCpuFeaturesGetSmmRegister (Index, SmmRegSmmBlocked);
  183. }
  184. if (DisabledCount != NULL) {
  185. *DisabledCount += (UINT32)SmmCpuFeaturesGetSmmRegister (Index, SmmRegSmmEnable);
  186. }
  187. }
  188. }
  189. }
  190. /**
  191. Checks if all CPUs (except Blocked & Disabled) have checked in for this SMI run
  192. @retval TRUE if all CPUs the have checked in.
  193. @retval FALSE if at least one Normal AP hasn't checked in.
  194. **/
  195. BOOLEAN
  196. AllCpusInSmmExceptBlockedDisabled (
  197. VOID
  198. )
  199. {
  200. UINT32 BlockedCount;
  201. UINT32 DisabledCount;
  202. BlockedCount = 0;
  203. DisabledCount = 0;
  204. //
  205. // Check to make sure mSmmMpSyncData->Counter is valid and not locked.
  206. //
  207. ASSERT (*mSmmMpSyncData->Counter <= mNumberOfCpus);
  208. //
  209. // Check whether all CPUs in SMM.
  210. //
  211. if (*mSmmMpSyncData->Counter == mNumberOfCpus) {
  212. return TRUE;
  213. }
  214. //
  215. // Check for the Blocked & Disabled Exceptions Case.
  216. //
  217. GetSmmDelayedBlockedDisabledCount (NULL, &BlockedCount, &DisabledCount);
  218. //
  219. // *mSmmMpSyncData->Counter might be updated by all APs concurrently. The value
  220. // can be dynamic changed. If some Aps enter the SMI after the BlockedCount &
  221. // DisabledCount check, then the *mSmmMpSyncData->Counter will be increased, thus
  222. // leading the *mSmmMpSyncData->Counter + BlockedCount + DisabledCount > mNumberOfCpus.
  223. // since the BlockedCount & DisabledCount are local variable, it's ok here only for
  224. // the checking of all CPUs In Smm.
  225. //
  226. if (*mSmmMpSyncData->Counter + BlockedCount + DisabledCount >= mNumberOfCpus) {
  227. return TRUE;
  228. }
  229. return FALSE;
  230. }
  231. /**
  232. Has OS enabled Lmce in the MSR_IA32_MCG_EXT_CTL
  233. @retval TRUE Os enable lmce.
  234. @retval FALSE Os not enable lmce.
  235. **/
  236. BOOLEAN
  237. IsLmceOsEnabled (
  238. VOID
  239. )
  240. {
  241. MSR_IA32_MCG_CAP_REGISTER McgCap;
  242. MSR_IA32_FEATURE_CONTROL_REGISTER FeatureCtrl;
  243. MSR_IA32_MCG_EXT_CTL_REGISTER McgExtCtrl;
  244. McgCap.Uint64 = AsmReadMsr64 (MSR_IA32_MCG_CAP);
  245. if (McgCap.Bits.MCG_LMCE_P == 0) {
  246. return FALSE;
  247. }
  248. FeatureCtrl.Uint64 = AsmReadMsr64 (MSR_IA32_FEATURE_CONTROL);
  249. if (FeatureCtrl.Bits.LmceOn == 0) {
  250. return FALSE;
  251. }
  252. McgExtCtrl.Uint64 = AsmReadMsr64 (MSR_IA32_MCG_EXT_CTL);
  253. return (BOOLEAN)(McgExtCtrl.Bits.LMCE_EN == 1);
  254. }
  255. /**
  256. Return if Local machine check exception signaled.
  257. Indicates (when set) that a local machine check exception was generated. This indicates that the current machine-check event was
  258. delivered to only the logical processor.
  259. @retval TRUE LMCE was signaled.
  260. @retval FALSE LMCE was not signaled.
  261. **/
  262. BOOLEAN
  263. IsLmceSignaled (
  264. VOID
  265. )
  266. {
  267. MSR_IA32_MCG_STATUS_REGISTER McgStatus;
  268. McgStatus.Uint64 = AsmReadMsr64 (MSR_IA32_MCG_STATUS);
  269. return (BOOLEAN)(McgStatus.Bits.LMCE_S == 1);
  270. }
  271. /**
  272. Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
  273. entering SMM, except SMI disabled APs.
  274. **/
  275. VOID
  276. SmmWaitForApArrival (
  277. VOID
  278. )
  279. {
  280. UINT64 Timer;
  281. UINTN Index;
  282. BOOLEAN LmceEn;
  283. BOOLEAN LmceSignal;
  284. UINT32 DelayedCount;
  285. UINT32 BlockedCount;
  286. DelayedCount = 0;
  287. BlockedCount = 0;
  288. ASSERT (*mSmmMpSyncData->Counter <= mNumberOfCpus);
  289. LmceEn = FALSE;
  290. LmceSignal = FALSE;
  291. if (mMachineCheckSupported) {
  292. LmceEn = IsLmceOsEnabled ();
  293. LmceSignal = IsLmceSignaled ();
  294. }
  295. //
  296. // Platform implementor should choose a timeout value appropriately:
  297. // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note
  298. // the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.
  299. // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI
  300. // and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will
  301. // be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the
  302. // SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.
  303. // - The timeout value must be longer than longest possible IO operation in the system
  304. //
  305. //
  306. // Sync with APs 1st timeout
  307. //
  308. for (Timer = StartSyncTimer ();
  309. !IsSyncTimerTimeout (Timer) && !(LmceEn && LmceSignal);
  310. )
  311. {
  312. mSmmMpSyncData->AllApArrivedWithException = AllCpusInSmmExceptBlockedDisabled ();
  313. if (mSmmMpSyncData->AllApArrivedWithException) {
  314. break;
  315. }
  316. CpuPause ();
  317. }
  318. //
  319. // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,
  320. // because:
  321. // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running
  322. // normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they
  323. // enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode
  324. // work while SMI handling is on-going.
  325. // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.
  326. // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state
  327. // will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal
  328. // mode work while SMI handling is on-going.
  329. // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:
  330. // - In traditional flow, SMI disabling is discouraged.
  331. // - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
  332. // In both cases, adding SMI-disabling checking code increases overhead.
  333. //
  334. if (*mSmmMpSyncData->Counter < mNumberOfCpus) {
  335. //
  336. // Send SMI IPIs to bring outside processors in
  337. //
  338. for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
  339. if (!(*(mSmmMpSyncData->CpuData[Index].Present)) && (gSmmCpuPrivate->ProcessorInfo[Index].ProcessorId != INVALID_APIC_ID)) {
  340. SendSmiIpi ((UINT32)gSmmCpuPrivate->ProcessorInfo[Index].ProcessorId);
  341. }
  342. }
  343. //
  344. // Sync with APs 2nd timeout.
  345. //
  346. for (Timer = StartSyncTimer ();
  347. !IsSyncTimerTimeout (Timer);
  348. )
  349. {
  350. mSmmMpSyncData->AllApArrivedWithException = AllCpusInSmmExceptBlockedDisabled ();
  351. if (mSmmMpSyncData->AllApArrivedWithException) {
  352. break;
  353. }
  354. CpuPause ();
  355. }
  356. }
  357. if (!mSmmMpSyncData->AllApArrivedWithException) {
  358. //
  359. // Check for the Blocked & Delayed Case.
  360. //
  361. GetSmmDelayedBlockedDisabledCount (&DelayedCount, &BlockedCount, NULL);
  362. DEBUG ((DEBUG_INFO, "SmmWaitForApArrival: Delayed AP Count = %d, Blocked AP Count = %d\n", DelayedCount, BlockedCount));
  363. }
  364. return;
  365. }
  366. /**
  367. Replace OS MTRR's with SMI MTRR's.
  368. @param CpuIndex Processor Index
  369. **/
  370. VOID
  371. ReplaceOSMtrrs (
  372. IN UINTN CpuIndex
  373. )
  374. {
  375. SmmCpuFeaturesDisableSmrr ();
  376. //
  377. // Replace all MTRRs registers
  378. //
  379. MtrrSetAllMtrrs (&gSmiMtrrs);
  380. }
  381. /**
  382. Wheck whether task has been finished by all APs.
  383. @param BlockMode Whether did it in block mode or non-block mode.
  384. @retval TRUE Task has been finished by all APs.
  385. @retval FALSE Task not has been finished by all APs.
  386. **/
  387. BOOLEAN
  388. WaitForAllAPsNotBusy (
  389. IN BOOLEAN BlockMode
  390. )
  391. {
  392. UINTN Index;
  393. for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
  394. //
  395. // Ignore BSP and APs which not call in SMM.
  396. //
  397. if (!IsPresentAp (Index)) {
  398. continue;
  399. }
  400. if (BlockMode) {
  401. AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
  402. ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
  403. } else {
  404. if (AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[Index].Busy)) {
  405. ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
  406. } else {
  407. return FALSE;
  408. }
  409. }
  410. }
  411. return TRUE;
  412. }
  413. /**
  414. Check whether it is an present AP.
  415. @param CpuIndex The AP index which calls this function.
  416. @retval TRUE It's a present AP.
  417. @retval TRUE This is not an AP or it is not present.
  418. **/
  419. BOOLEAN
  420. IsPresentAp (
  421. IN UINTN CpuIndex
  422. )
  423. {
  424. return ((CpuIndex != gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu) &&
  425. *(mSmmMpSyncData->CpuData[CpuIndex].Present));
  426. }
  427. /**
  428. Clean up the status flags used during executing the procedure.
  429. @param CpuIndex The AP index which calls this function.
  430. **/
  431. VOID
  432. ReleaseToken (
  433. IN UINTN CpuIndex
  434. )
  435. {
  436. PROCEDURE_TOKEN *Token;
  437. Token = mSmmMpSyncData->CpuData[CpuIndex].Token;
  438. if (InterlockedDecrement (&Token->RunningApCount) == 0) {
  439. ReleaseSpinLock (Token->SpinLock);
  440. }
  441. mSmmMpSyncData->CpuData[CpuIndex].Token = NULL;
  442. }
  443. /**
  444. Free the tokens in the maintained list.
  445. **/
  446. VOID
  447. ResetTokens (
  448. VOID
  449. )
  450. {
  451. //
  452. // Reset the FirstFreeToken to the beginning of token list upon exiting SMI.
  453. //
  454. gSmmCpuPrivate->FirstFreeToken = GetFirstNode (&gSmmCpuPrivate->TokenList);
  455. }
  456. /**
  457. SMI handler for BSP.
  458. @param CpuIndex BSP processor Index
  459. @param SyncMode SMM MP sync mode
  460. **/
  461. VOID
  462. BSPHandler (
  463. IN UINTN CpuIndex,
  464. IN SMM_CPU_SYNC_MODE SyncMode
  465. )
  466. {
  467. UINTN Index;
  468. MTRR_SETTINGS Mtrrs;
  469. UINTN ApCount;
  470. BOOLEAN ClearTopLevelSmiResult;
  471. UINTN PresentCount;
  472. ASSERT (CpuIndex == mSmmMpSyncData->BspIndex);
  473. ApCount = 0;
  474. //
  475. // Flag BSP's presence
  476. //
  477. *mSmmMpSyncData->InsideSmm = TRUE;
  478. //
  479. // Initialize Debug Agent to start source level debug in BSP handler
  480. //
  481. InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI, NULL, NULL);
  482. //
  483. // Mark this processor's presence
  484. //
  485. *(mSmmMpSyncData->CpuData[CpuIndex].Present) = TRUE;
  486. //
  487. // Clear platform top level SMI status bit before calling SMI handlers. If
  488. // we cleared it after SMI handlers are run, we would miss the SMI that
  489. // occurs after SMI handlers are done and before SMI status bit is cleared.
  490. //
  491. ClearTopLevelSmiResult = ClearTopLevelSmiStatus ();
  492. ASSERT (ClearTopLevelSmiResult == TRUE);
  493. //
  494. // Set running processor index
  495. //
  496. gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu = CpuIndex;
  497. //
  498. // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
  499. //
  500. if ((SyncMode == SmmCpuSyncModeTradition) || SmmCpuFeaturesNeedConfigureMtrrs ()) {
  501. //
  502. // Wait for APs to arrive
  503. //
  504. SmmWaitForApArrival ();
  505. //
  506. // Lock the counter down and retrieve the number of APs
  507. //
  508. *mSmmMpSyncData->AllCpusInSync = TRUE;
  509. ApCount = LockdownSemaphore (mSmmMpSyncData->Counter) - 1;
  510. //
  511. // Wait for all APs to get ready for programming MTRRs
  512. //
  513. WaitForAllAPs (ApCount);
  514. if (SmmCpuFeaturesNeedConfigureMtrrs ()) {
  515. //
  516. // Signal all APs it's time for backup MTRRs
  517. //
  518. ReleaseAllAPs ();
  519. //
  520. // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
  521. // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
  522. // to a large enough value to avoid this situation.
  523. // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
  524. // We do the backup first and then set MTRR to avoid race condition for threads
  525. // in the same core.
  526. //
  527. MtrrGetAllMtrrs (&Mtrrs);
  528. //
  529. // Wait for all APs to complete their MTRR saving
  530. //
  531. WaitForAllAPs (ApCount);
  532. //
  533. // Let all processors program SMM MTRRs together
  534. //
  535. ReleaseAllAPs ();
  536. //
  537. // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
  538. // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
  539. // to a large enough value to avoid this situation.
  540. //
  541. ReplaceOSMtrrs (CpuIndex);
  542. //
  543. // Wait for all APs to complete their MTRR programming
  544. //
  545. WaitForAllAPs (ApCount);
  546. }
  547. }
  548. //
  549. // The BUSY lock is initialized to Acquired state
  550. //
  551. AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
  552. //
  553. // Perform the pre tasks
  554. //
  555. PerformPreTasks ();
  556. //
  557. // Invoke SMM Foundation EntryPoint with the processor information context.
  558. //
  559. gSmmCpuPrivate->SmmCoreEntry (&gSmmCpuPrivate->SmmCoreEntryContext);
  560. //
  561. // Make sure all APs have completed their pending none-block tasks
  562. //
  563. WaitForAllAPsNotBusy (TRUE);
  564. //
  565. // Perform the remaining tasks
  566. //
  567. PerformRemainingTasks ();
  568. //
  569. // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
  570. // make those APs to exit SMI synchronously. APs which arrive later will be excluded and
  571. // will run through freely.
  572. //
  573. if ((SyncMode != SmmCpuSyncModeTradition) && !SmmCpuFeaturesNeedConfigureMtrrs ()) {
  574. //
  575. // Lock the counter down and retrieve the number of APs
  576. //
  577. *mSmmMpSyncData->AllCpusInSync = TRUE;
  578. ApCount = LockdownSemaphore (mSmmMpSyncData->Counter) - 1;
  579. //
  580. // Make sure all APs have their Present flag set
  581. //
  582. while (TRUE) {
  583. PresentCount = 0;
  584. for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
  585. if (*(mSmmMpSyncData->CpuData[Index].Present)) {
  586. PresentCount++;
  587. }
  588. }
  589. if (PresentCount > ApCount) {
  590. break;
  591. }
  592. }
  593. }
  594. //
  595. // Notify all APs to exit
  596. //
  597. *mSmmMpSyncData->InsideSmm = FALSE;
  598. ReleaseAllAPs ();
  599. //
  600. // Wait for all APs to complete their pending tasks
  601. //
  602. WaitForAllAPs (ApCount);
  603. if (SmmCpuFeaturesNeedConfigureMtrrs ()) {
  604. //
  605. // Signal APs to restore MTRRs
  606. //
  607. ReleaseAllAPs ();
  608. //
  609. // Restore OS MTRRs
  610. //
  611. SmmCpuFeaturesReenableSmrr ();
  612. MtrrSetAllMtrrs (&Mtrrs);
  613. //
  614. // Wait for all APs to complete MTRR programming
  615. //
  616. WaitForAllAPs (ApCount);
  617. }
  618. //
  619. // Stop source level debug in BSP handler, the code below will not be
  620. // debugged.
  621. //
  622. InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI, NULL, NULL);
  623. //
  624. // Signal APs to Reset states/semaphore for this processor
  625. //
  626. ReleaseAllAPs ();
  627. //
  628. // Perform pending operations for hot-plug
  629. //
  630. SmmCpuUpdate ();
  631. //
  632. // Clear the Present flag of BSP
  633. //
  634. *(mSmmMpSyncData->CpuData[CpuIndex].Present) = FALSE;
  635. //
  636. // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
  637. // WaitForAllAps does not depend on the Present flag.
  638. //
  639. WaitForAllAPs (ApCount);
  640. //
  641. // Reset the tokens buffer.
  642. //
  643. ResetTokens ();
  644. //
  645. // Reset BspIndex to -1, meaning BSP has not been elected.
  646. //
  647. if (FeaturePcdGet (PcdCpuSmmEnableBspElection)) {
  648. mSmmMpSyncData->BspIndex = (UINT32)-1;
  649. }
  650. //
  651. // Allow APs to check in from this point on
  652. //
  653. *mSmmMpSyncData->Counter = 0;
  654. *mSmmMpSyncData->AllCpusInSync = FALSE;
  655. mSmmMpSyncData->AllApArrivedWithException = FALSE;
  656. }
  657. /**
  658. SMI handler for AP.
  659. @param CpuIndex AP processor Index.
  660. @param ValidSmi Indicates that current SMI is a valid SMI or not.
  661. @param SyncMode SMM MP sync mode.
  662. **/
  663. VOID
  664. APHandler (
  665. IN UINTN CpuIndex,
  666. IN BOOLEAN ValidSmi,
  667. IN SMM_CPU_SYNC_MODE SyncMode
  668. )
  669. {
  670. UINT64 Timer;
  671. UINTN BspIndex;
  672. MTRR_SETTINGS Mtrrs;
  673. EFI_STATUS ProcedureStatus;
  674. //
  675. // Timeout BSP
  676. //
  677. for (Timer = StartSyncTimer ();
  678. !IsSyncTimerTimeout (Timer) &&
  679. !(*mSmmMpSyncData->InsideSmm);
  680. )
  681. {
  682. CpuPause ();
  683. }
  684. if (!(*mSmmMpSyncData->InsideSmm)) {
  685. //
  686. // BSP timeout in the first round
  687. //
  688. if (mSmmMpSyncData->BspIndex != -1) {
  689. //
  690. // BSP Index is known
  691. // Existing AP is in SMI now but BSP not in, so, try bring BSP in SMM.
  692. //
  693. BspIndex = mSmmMpSyncData->BspIndex;
  694. ASSERT (CpuIndex != BspIndex);
  695. //
  696. // Send SMI IPI to bring BSP in
  697. //
  698. SendSmiIpi ((UINT32)gSmmCpuPrivate->ProcessorInfo[BspIndex].ProcessorId);
  699. //
  700. // Now clock BSP for the 2nd time
  701. //
  702. for (Timer = StartSyncTimer ();
  703. !IsSyncTimerTimeout (Timer) &&
  704. !(*mSmmMpSyncData->InsideSmm);
  705. )
  706. {
  707. CpuPause ();
  708. }
  709. if (!(*mSmmMpSyncData->InsideSmm)) {
  710. //
  711. // Give up since BSP is unable to enter SMM
  712. // and signal the completion of this AP
  713. // Reduce the mSmmMpSyncData->Counter!
  714. //
  715. WaitForSemaphore (mSmmMpSyncData->Counter);
  716. return;
  717. }
  718. } else {
  719. //
  720. // Don't know BSP index. Give up without sending IPI to BSP.
  721. // Reduce the mSmmMpSyncData->Counter!
  722. //
  723. WaitForSemaphore (mSmmMpSyncData->Counter);
  724. return;
  725. }
  726. }
  727. //
  728. // BSP is available
  729. //
  730. BspIndex = mSmmMpSyncData->BspIndex;
  731. ASSERT (CpuIndex != BspIndex);
  732. //
  733. // Mark this processor's presence
  734. //
  735. *(mSmmMpSyncData->CpuData[CpuIndex].Present) = TRUE;
  736. if ((SyncMode == SmmCpuSyncModeTradition) || SmmCpuFeaturesNeedConfigureMtrrs ()) {
  737. //
  738. // Notify BSP of arrival at this point
  739. //
  740. ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
  741. }
  742. if (SmmCpuFeaturesNeedConfigureMtrrs ()) {
  743. //
  744. // Wait for the signal from BSP to backup MTRRs
  745. //
  746. WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
  747. //
  748. // Backup OS MTRRs
  749. //
  750. MtrrGetAllMtrrs (&Mtrrs);
  751. //
  752. // Signal BSP the completion of this AP
  753. //
  754. ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
  755. //
  756. // Wait for BSP's signal to program MTRRs
  757. //
  758. WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
  759. //
  760. // Replace OS MTRRs with SMI MTRRs
  761. //
  762. ReplaceOSMtrrs (CpuIndex);
  763. //
  764. // Signal BSP the completion of this AP
  765. //
  766. ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
  767. }
  768. while (TRUE) {
  769. //
  770. // Wait for something to happen
  771. //
  772. WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
  773. //
  774. // Check if BSP wants to exit SMM
  775. //
  776. if (!(*mSmmMpSyncData->InsideSmm)) {
  777. break;
  778. }
  779. //
  780. // BUSY should be acquired by SmmStartupThisAp()
  781. //
  782. ASSERT (
  783. !AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[CpuIndex].Busy)
  784. );
  785. //
  786. // Invoke the scheduled procedure
  787. //
  788. ProcedureStatus = (*mSmmMpSyncData->CpuData[CpuIndex].Procedure)(
  789. (VOID *)mSmmMpSyncData->CpuData[CpuIndex].Parameter
  790. );
  791. if (mSmmMpSyncData->CpuData[CpuIndex].Status != NULL) {
  792. *mSmmMpSyncData->CpuData[CpuIndex].Status = ProcedureStatus;
  793. }
  794. if (mSmmMpSyncData->CpuData[CpuIndex].Token != NULL) {
  795. ReleaseToken (CpuIndex);
  796. }
  797. //
  798. // Release BUSY
  799. //
  800. ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
  801. }
  802. if (SmmCpuFeaturesNeedConfigureMtrrs ()) {
  803. //
  804. // Notify BSP the readiness of this AP to program MTRRs
  805. //
  806. ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
  807. //
  808. // Wait for the signal from BSP to program MTRRs
  809. //
  810. WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
  811. //
  812. // Restore OS MTRRs
  813. //
  814. SmmCpuFeaturesReenableSmrr ();
  815. MtrrSetAllMtrrs (&Mtrrs);
  816. }
  817. //
  818. // Notify BSP the readiness of this AP to Reset states/semaphore for this processor
  819. //
  820. ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
  821. //
  822. // Wait for the signal from BSP to Reset states/semaphore for this processor
  823. //
  824. WaitForSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
  825. //
  826. // Reset states/semaphore for this processor
  827. //
  828. *(mSmmMpSyncData->CpuData[CpuIndex].Present) = FALSE;
  829. //
  830. // Notify BSP the readiness of this AP to exit SMM
  831. //
  832. ReleaseSemaphore (mSmmMpSyncData->CpuData[BspIndex].Run);
  833. }
  834. /**
  835. Create 4G PageTable in SMRAM.
  836. @param[in] Is32BitPageTable Whether the page table is 32-bit PAE
  837. @return PageTable Address
  838. **/
  839. UINT32
  840. Gen4GPageTable (
  841. IN BOOLEAN Is32BitPageTable
  842. )
  843. {
  844. VOID *PageTable;
  845. UINTN Index;
  846. UINT64 *Pte;
  847. UINTN PagesNeeded;
  848. UINTN Low2MBoundary;
  849. UINTN High2MBoundary;
  850. UINTN Pages;
  851. UINTN GuardPage;
  852. UINT64 *Pdpte;
  853. UINTN PageIndex;
  854. UINTN PageAddress;
  855. Low2MBoundary = 0;
  856. High2MBoundary = 0;
  857. PagesNeeded = 0;
  858. if (FeaturePcdGet (PcdCpuSmmStackGuard)) {
  859. //
  860. // Add one more page for known good stack, then find the lower 2MB aligned address.
  861. //
  862. Low2MBoundary = (mSmmStackArrayBase + EFI_PAGE_SIZE) & ~(SIZE_2MB-1);
  863. //
  864. // Add two more pages for known good stack and stack guard page,
  865. // then find the lower 2MB aligned address.
  866. //
  867. High2MBoundary = (mSmmStackArrayEnd - mSmmStackSize - mSmmShadowStackSize + EFI_PAGE_SIZE * 2) & ~(SIZE_2MB-1);
  868. PagesNeeded = ((High2MBoundary - Low2MBoundary) / SIZE_2MB) + 1;
  869. }
  870. //
  871. // Allocate the page table
  872. //
  873. PageTable = AllocatePageTableMemory (5 + PagesNeeded);
  874. ASSERT (PageTable != NULL);
  875. PageTable = (VOID *)((UINTN)PageTable);
  876. Pte = (UINT64 *)PageTable;
  877. //
  878. // Zero out all page table entries first
  879. //
  880. ZeroMem (Pte, EFI_PAGES_TO_SIZE (1));
  881. //
  882. // Set Page Directory Pointers
  883. //
  884. for (Index = 0; Index < 4; Index++) {
  885. Pte[Index] = ((UINTN)PageTable + EFI_PAGE_SIZE * (Index + 1)) | mAddressEncMask |
  886. (Is32BitPageTable ? IA32_PAE_PDPTE_ATTRIBUTE_BITS : PAGE_ATTRIBUTE_BITS);
  887. }
  888. Pte += EFI_PAGE_SIZE / sizeof (*Pte);
  889. //
  890. // Fill in Page Directory Entries
  891. //
  892. for (Index = 0; Index < EFI_PAGE_SIZE * 4 / sizeof (*Pte); Index++) {
  893. Pte[Index] = (Index << 21) | mAddressEncMask | IA32_PG_PS | PAGE_ATTRIBUTE_BITS;
  894. }
  895. Pdpte = (UINT64 *)PageTable;
  896. if (FeaturePcdGet (PcdCpuSmmStackGuard)) {
  897. Pages = (UINTN)PageTable + EFI_PAGES_TO_SIZE (5);
  898. GuardPage = mSmmStackArrayBase + EFI_PAGE_SIZE;
  899. for (PageIndex = Low2MBoundary; PageIndex <= High2MBoundary; PageIndex += SIZE_2MB) {
  900. Pte = (UINT64 *)(UINTN)(Pdpte[BitFieldRead32 ((UINT32)PageIndex, 30, 31)] & ~mAddressEncMask & ~(EFI_PAGE_SIZE - 1));
  901. Pte[BitFieldRead32 ((UINT32)PageIndex, 21, 29)] = (UINT64)Pages | mAddressEncMask | PAGE_ATTRIBUTE_BITS;
  902. //
  903. // Fill in Page Table Entries
  904. //
  905. Pte = (UINT64 *)Pages;
  906. PageAddress = PageIndex;
  907. for (Index = 0; Index < EFI_PAGE_SIZE / sizeof (*Pte); Index++) {
  908. if (PageAddress == GuardPage) {
  909. //
  910. // Mark the guard page as non-present
  911. //
  912. Pte[Index] = PageAddress | mAddressEncMask;
  913. GuardPage += (mSmmStackSize + mSmmShadowStackSize);
  914. if (GuardPage > mSmmStackArrayEnd) {
  915. GuardPage = 0;
  916. }
  917. } else {
  918. Pte[Index] = PageAddress | mAddressEncMask | PAGE_ATTRIBUTE_BITS;
  919. }
  920. PageAddress += EFI_PAGE_SIZE;
  921. }
  922. Pages += EFI_PAGE_SIZE;
  923. }
  924. }
  925. if ((PcdGet8 (PcdNullPointerDetectionPropertyMask) & BIT1) != 0) {
  926. Pte = (UINT64 *)(UINTN)(Pdpte[0] & ~mAddressEncMask & ~(EFI_PAGE_SIZE - 1));
  927. if ((Pte[0] & IA32_PG_PS) == 0) {
  928. // 4K-page entries are already mapped. Just hide the first one anyway.
  929. Pte = (UINT64 *)(UINTN)(Pte[0] & ~mAddressEncMask & ~(EFI_PAGE_SIZE - 1));
  930. Pte[0] &= ~(UINT64)IA32_PG_P; // Hide page 0
  931. } else {
  932. // Create 4K-page entries
  933. Pages = (UINTN)AllocatePageTableMemory (1);
  934. ASSERT (Pages != 0);
  935. Pte[0] = (UINT64)(Pages | mAddressEncMask | PAGE_ATTRIBUTE_BITS);
  936. Pte = (UINT64 *)Pages;
  937. PageAddress = 0;
  938. Pte[0] = PageAddress | mAddressEncMask; // Hide page 0 but present left
  939. for (Index = 1; Index < EFI_PAGE_SIZE / sizeof (*Pte); Index++) {
  940. PageAddress += EFI_PAGE_SIZE;
  941. Pte[Index] = PageAddress | mAddressEncMask | PAGE_ATTRIBUTE_BITS;
  942. }
  943. }
  944. }
  945. return (UINT32)(UINTN)PageTable;
  946. }
  947. /**
  948. Checks whether the input token is the current used token.
  949. @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
  950. BroadcastProcedure.
  951. @retval TRUE The input token is the current used token.
  952. @retval FALSE The input token is not the current used token.
  953. **/
  954. BOOLEAN
  955. IsTokenInUse (
  956. IN SPIN_LOCK *Token
  957. )
  958. {
  959. LIST_ENTRY *Link;
  960. PROCEDURE_TOKEN *ProcToken;
  961. if (Token == NULL) {
  962. return FALSE;
  963. }
  964. Link = GetFirstNode (&gSmmCpuPrivate->TokenList);
  965. //
  966. // Only search used tokens.
  967. //
  968. while (Link != gSmmCpuPrivate->FirstFreeToken) {
  969. ProcToken = PROCEDURE_TOKEN_FROM_LINK (Link);
  970. if (ProcToken->SpinLock == Token) {
  971. return TRUE;
  972. }
  973. Link = GetNextNode (&gSmmCpuPrivate->TokenList, Link);
  974. }
  975. return FALSE;
  976. }
  977. /**
  978. Allocate buffer for the SPIN_LOCK and PROCEDURE_TOKEN.
  979. @return First token of the token buffer.
  980. **/
  981. LIST_ENTRY *
  982. AllocateTokenBuffer (
  983. VOID
  984. )
  985. {
  986. UINTN SpinLockSize;
  987. UINT32 TokenCountPerChunk;
  988. UINTN Index;
  989. SPIN_LOCK *SpinLock;
  990. UINT8 *SpinLockBuffer;
  991. PROCEDURE_TOKEN *ProcTokens;
  992. SpinLockSize = GetSpinLockProperties ();
  993. TokenCountPerChunk = FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk);
  994. ASSERT (TokenCountPerChunk != 0);
  995. if (TokenCountPerChunk == 0) {
  996. DEBUG ((DEBUG_ERROR, "PcdCpuSmmMpTokenCountPerChunk should not be Zero!\n"));
  997. CpuDeadLoop ();
  998. }
  999. DEBUG ((DEBUG_INFO, "CpuSmm: SpinLock Size = 0x%x, PcdCpuSmmMpTokenCountPerChunk = 0x%x\n", SpinLockSize, TokenCountPerChunk));
  1000. //
  1001. // Separate the Spin_lock and Proc_token because the alignment requires by Spin_Lock.
  1002. //
  1003. SpinLockBuffer = AllocatePool (SpinLockSize * TokenCountPerChunk);
  1004. ASSERT (SpinLockBuffer != NULL);
  1005. ProcTokens = AllocatePool (sizeof (PROCEDURE_TOKEN) * TokenCountPerChunk);
  1006. ASSERT (ProcTokens != NULL);
  1007. for (Index = 0; Index < TokenCountPerChunk; Index++) {
  1008. SpinLock = (SPIN_LOCK *)(SpinLockBuffer + SpinLockSize * Index);
  1009. InitializeSpinLock (SpinLock);
  1010. ProcTokens[Index].Signature = PROCEDURE_TOKEN_SIGNATURE;
  1011. ProcTokens[Index].SpinLock = SpinLock;
  1012. ProcTokens[Index].RunningApCount = 0;
  1013. InsertTailList (&gSmmCpuPrivate->TokenList, &ProcTokens[Index].Link);
  1014. }
  1015. return &ProcTokens[0].Link;
  1016. }
  1017. /**
  1018. Get the free token.
  1019. If no free token, allocate new tokens then return the free one.
  1020. @param RunningApsCount The Running Aps count for this token.
  1021. @retval return the first free PROCEDURE_TOKEN.
  1022. **/
  1023. PROCEDURE_TOKEN *
  1024. GetFreeToken (
  1025. IN UINT32 RunningApsCount
  1026. )
  1027. {
  1028. PROCEDURE_TOKEN *NewToken;
  1029. //
  1030. // If FirstFreeToken meets the end of token list, enlarge the token list.
  1031. // Set FirstFreeToken to the first free token.
  1032. //
  1033. if (gSmmCpuPrivate->FirstFreeToken == &gSmmCpuPrivate->TokenList) {
  1034. gSmmCpuPrivate->FirstFreeToken = AllocateTokenBuffer ();
  1035. }
  1036. NewToken = PROCEDURE_TOKEN_FROM_LINK (gSmmCpuPrivate->FirstFreeToken);
  1037. gSmmCpuPrivate->FirstFreeToken = GetNextNode (&gSmmCpuPrivate->TokenList, gSmmCpuPrivate->FirstFreeToken);
  1038. NewToken->RunningApCount = RunningApsCount;
  1039. AcquireSpinLock (NewToken->SpinLock);
  1040. return NewToken;
  1041. }
  1042. /**
  1043. Checks status of specified AP.
  1044. This function checks whether the specified AP has finished the task assigned
  1045. by StartupThisAP(), and whether timeout expires.
  1046. @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
  1047. BroadcastProcedure.
  1048. @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
  1049. @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
  1050. **/
  1051. EFI_STATUS
  1052. IsApReady (
  1053. IN SPIN_LOCK *Token
  1054. )
  1055. {
  1056. if (AcquireSpinLockOrFail (Token)) {
  1057. ReleaseSpinLock (Token);
  1058. return EFI_SUCCESS;
  1059. }
  1060. return EFI_NOT_READY;
  1061. }
  1062. /**
  1063. Schedule a procedure to run on the specified CPU.
  1064. @param[in] Procedure The address of the procedure to run
  1065. @param[in] CpuIndex Target CPU Index
  1066. @param[in,out] ProcArguments The parameter to pass to the procedure
  1067. @param[in] Token This is an optional parameter that allows the caller to execute the
  1068. procedure in a blocking or non-blocking fashion. If it is NULL the
  1069. call is blocking, and the call will not return until the AP has
  1070. completed the procedure. If the token is not NULL, the call will
  1071. return immediately. The caller can check whether the procedure has
  1072. completed with CheckOnProcedure or WaitForProcedure.
  1073. @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for the APs to finish
  1074. execution of Procedure, either for blocking or non-blocking mode.
  1075. Zero means infinity. If the timeout expires before all APs return
  1076. from Procedure, then Procedure on the failed APs is terminated. If
  1077. the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
  1078. If the timeout expires in non-blocking mode, the timeout determined
  1079. can be through CheckOnProcedure or WaitForProcedure.
  1080. Note that timeout support is optional. Whether an implementation
  1081. supports this feature can be determined via the Attributes data
  1082. member.
  1083. @param[in,out] CpuStatus This optional pointer may be used to get the status code returned
  1084. by Procedure when it completes execution on the target AP, or with
  1085. EFI_TIMEOUT if the Procedure fails to complete within the optional
  1086. timeout. The implementation will update this variable with
  1087. EFI_NOT_READY prior to starting Procedure on the target AP.
  1088. @retval EFI_INVALID_PARAMETER CpuNumber not valid
  1089. @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
  1090. @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
  1091. @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
  1092. @retval EFI_SUCCESS The procedure has been successfully scheduled
  1093. **/
  1094. EFI_STATUS
  1095. InternalSmmStartupThisAp (
  1096. IN EFI_AP_PROCEDURE2 Procedure,
  1097. IN UINTN CpuIndex,
  1098. IN OUT VOID *ProcArguments OPTIONAL,
  1099. IN MM_COMPLETION *Token,
  1100. IN UINTN TimeoutInMicroseconds,
  1101. IN OUT EFI_STATUS *CpuStatus
  1102. )
  1103. {
  1104. PROCEDURE_TOKEN *ProcToken;
  1105. if (CpuIndex >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus) {
  1106. DEBUG ((DEBUG_ERROR, "CpuIndex(%d) >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus(%d)\n", CpuIndex, gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus));
  1107. return EFI_INVALID_PARAMETER;
  1108. }
  1109. if (CpuIndex == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu) {
  1110. DEBUG ((DEBUG_ERROR, "CpuIndex(%d) == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu\n", CpuIndex));
  1111. return EFI_INVALID_PARAMETER;
  1112. }
  1113. if (gSmmCpuPrivate->ProcessorInfo[CpuIndex].ProcessorId == INVALID_APIC_ID) {
  1114. return EFI_INVALID_PARAMETER;
  1115. }
  1116. if (!(*(mSmmMpSyncData->CpuData[CpuIndex].Present))) {
  1117. if (mSmmMpSyncData->EffectiveSyncMode == SmmCpuSyncModeTradition) {
  1118. DEBUG ((DEBUG_ERROR, "!mSmmMpSyncData->CpuData[%d].Present\n", CpuIndex));
  1119. }
  1120. return EFI_INVALID_PARAMETER;
  1121. }
  1122. if (gSmmCpuPrivate->Operation[CpuIndex] == SmmCpuRemove) {
  1123. if (!FeaturePcdGet (PcdCpuHotPlugSupport)) {
  1124. DEBUG ((DEBUG_ERROR, "gSmmCpuPrivate->Operation[%d] == SmmCpuRemove\n", CpuIndex));
  1125. }
  1126. return EFI_INVALID_PARAMETER;
  1127. }
  1128. if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes & EFI_MM_MP_TIMEOUT_SUPPORTED) == 0)) {
  1129. return EFI_INVALID_PARAMETER;
  1130. }
  1131. if (Procedure == NULL) {
  1132. return EFI_INVALID_PARAMETER;
  1133. }
  1134. AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
  1135. mSmmMpSyncData->CpuData[CpuIndex].Procedure = Procedure;
  1136. mSmmMpSyncData->CpuData[CpuIndex].Parameter = ProcArguments;
  1137. if (Token != NULL) {
  1138. if (Token != &mSmmStartupThisApToken) {
  1139. //
  1140. // When Token points to mSmmStartupThisApToken, this routine is called
  1141. // from SmmStartupThisAp() in non-blocking mode (PcdCpuSmmBlockStartupThisAp == FALSE).
  1142. //
  1143. // In this case, caller wants to startup AP procedure in non-blocking
  1144. // mode and cannot get the completion status from the Token because there
  1145. // is no way to return the Token to caller from SmmStartupThisAp().
  1146. // Caller needs to use its implementation specific way to query the completion status.
  1147. //
  1148. // There is no need to allocate a token for such case so the 3 overheads
  1149. // can be avoided:
  1150. // 1. Call AllocateTokenBuffer() when there is no free token.
  1151. // 2. Get a free token from the token buffer.
  1152. // 3. Call ReleaseToken() in APHandler().
  1153. //
  1154. ProcToken = GetFreeToken (1);
  1155. mSmmMpSyncData->CpuData[CpuIndex].Token = ProcToken;
  1156. *Token = (MM_COMPLETION)ProcToken->SpinLock;
  1157. }
  1158. }
  1159. mSmmMpSyncData->CpuData[CpuIndex].Status = CpuStatus;
  1160. if (mSmmMpSyncData->CpuData[CpuIndex].Status != NULL) {
  1161. *mSmmMpSyncData->CpuData[CpuIndex].Status = EFI_NOT_READY;
  1162. }
  1163. ReleaseSemaphore (mSmmMpSyncData->CpuData[CpuIndex].Run);
  1164. if (Token == NULL) {
  1165. AcquireSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
  1166. ReleaseSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
  1167. }
  1168. return EFI_SUCCESS;
  1169. }
  1170. /**
  1171. Worker function to execute a caller provided function on all enabled APs.
  1172. @param[in] Procedure A pointer to the function to be run on
  1173. enabled APs of the system.
  1174. @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
  1175. APs to return from Procedure, either for
  1176. blocking or non-blocking mode.
  1177. @param[in,out] ProcedureArguments The parameter passed into Procedure for
  1178. all APs.
  1179. @param[in,out] Token This is an optional parameter that allows the caller to execute the
  1180. procedure in a blocking or non-blocking fashion. If it is NULL the
  1181. call is blocking, and the call will not return until the AP has
  1182. completed the procedure. If the token is not NULL, the call will
  1183. return immediately. The caller can check whether the procedure has
  1184. completed with CheckOnProcedure or WaitForProcedure.
  1185. @param[in,out] CPUStatus This optional pointer may be used to get the status code returned
  1186. by Procedure when it completes execution on the target AP, or with
  1187. EFI_TIMEOUT if the Procedure fails to complete within the optional
  1188. timeout. The implementation will update this variable with
  1189. EFI_NOT_READY prior to starting Procedure on the target AP.
  1190. @retval EFI_SUCCESS In blocking mode, all APs have finished before
  1191. the timeout expired.
  1192. @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
  1193. to all enabled APs.
  1194. @retval others Failed to Startup all APs.
  1195. **/
  1196. EFI_STATUS
  1197. InternalSmmStartupAllAPs (
  1198. IN EFI_AP_PROCEDURE2 Procedure,
  1199. IN UINTN TimeoutInMicroseconds,
  1200. IN OUT VOID *ProcedureArguments OPTIONAL,
  1201. IN OUT MM_COMPLETION *Token,
  1202. IN OUT EFI_STATUS *CPUStatus
  1203. )
  1204. {
  1205. UINTN Index;
  1206. UINTN CpuCount;
  1207. PROCEDURE_TOKEN *ProcToken;
  1208. if ((TimeoutInMicroseconds != 0) && ((mSmmMp.Attributes & EFI_MM_MP_TIMEOUT_SUPPORTED) == 0)) {
  1209. return EFI_INVALID_PARAMETER;
  1210. }
  1211. if (Procedure == NULL) {
  1212. return EFI_INVALID_PARAMETER;
  1213. }
  1214. CpuCount = 0;
  1215. for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
  1216. if (IsPresentAp (Index)) {
  1217. CpuCount++;
  1218. if (gSmmCpuPrivate->Operation[Index] == SmmCpuRemove) {
  1219. return EFI_INVALID_PARAMETER;
  1220. }
  1221. if (!AcquireSpinLockOrFail (mSmmMpSyncData->CpuData[Index].Busy)) {
  1222. return EFI_NOT_READY;
  1223. }
  1224. ReleaseSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
  1225. }
  1226. }
  1227. if (CpuCount == 0) {
  1228. return EFI_NOT_STARTED;
  1229. }
  1230. if (Token != NULL) {
  1231. ProcToken = GetFreeToken ((UINT32)mMaxNumberOfCpus);
  1232. *Token = (MM_COMPLETION)ProcToken->SpinLock;
  1233. } else {
  1234. ProcToken = NULL;
  1235. }
  1236. //
  1237. // Make sure all BUSY should be acquired.
  1238. //
  1239. // Because former code already check mSmmMpSyncData->CpuData[***].Busy for each AP.
  1240. // Here code always use AcquireSpinLock instead of AcquireSpinLockOrFail for not
  1241. // block mode.
  1242. //
  1243. for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
  1244. if (IsPresentAp (Index)) {
  1245. AcquireSpinLock (mSmmMpSyncData->CpuData[Index].Busy);
  1246. }
  1247. }
  1248. for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
  1249. if (IsPresentAp (Index)) {
  1250. mSmmMpSyncData->CpuData[Index].Procedure = (EFI_AP_PROCEDURE2)Procedure;
  1251. mSmmMpSyncData->CpuData[Index].Parameter = ProcedureArguments;
  1252. if (ProcToken != NULL) {
  1253. mSmmMpSyncData->CpuData[Index].Token = ProcToken;
  1254. }
  1255. if (CPUStatus != NULL) {
  1256. mSmmMpSyncData->CpuData[Index].Status = &CPUStatus[Index];
  1257. if (mSmmMpSyncData->CpuData[Index].Status != NULL) {
  1258. *mSmmMpSyncData->CpuData[Index].Status = EFI_NOT_READY;
  1259. }
  1260. }
  1261. } else {
  1262. //
  1263. // PI spec requirement:
  1264. // For every excluded processor, the array entry must contain a value of EFI_NOT_STARTED.
  1265. //
  1266. if (CPUStatus != NULL) {
  1267. CPUStatus[Index] = EFI_NOT_STARTED;
  1268. }
  1269. //
  1270. // Decrease the count to mark this processor(AP or BSP) as finished.
  1271. //
  1272. if (ProcToken != NULL) {
  1273. WaitForSemaphore (&ProcToken->RunningApCount);
  1274. }
  1275. }
  1276. }
  1277. ReleaseAllAPs ();
  1278. if (Token == NULL) {
  1279. //
  1280. // Make sure all APs have completed their tasks.
  1281. //
  1282. WaitForAllAPsNotBusy (TRUE);
  1283. }
  1284. return EFI_SUCCESS;
  1285. }
  1286. /**
  1287. ISO C99 6.5.2.2 "Function calls", paragraph 9:
  1288. If the function is defined with a type that is not compatible with
  1289. the type (of the expression) pointed to by the expression that
  1290. denotes the called function, the behavior is undefined.
  1291. So add below wrapper function to convert between EFI_AP_PROCEDURE
  1292. and EFI_AP_PROCEDURE2.
  1293. Wrapper for Procedures.
  1294. @param[in] Buffer Pointer to PROCEDURE_WRAPPER buffer.
  1295. **/
  1296. EFI_STATUS
  1297. EFIAPI
  1298. ProcedureWrapper (
  1299. IN VOID *Buffer
  1300. )
  1301. {
  1302. PROCEDURE_WRAPPER *Wrapper;
  1303. Wrapper = Buffer;
  1304. Wrapper->Procedure (Wrapper->ProcedureArgument);
  1305. return EFI_SUCCESS;
  1306. }
  1307. /**
  1308. Schedule a procedure to run on the specified CPU in blocking mode.
  1309. @param[in] Procedure The address of the procedure to run
  1310. @param[in] CpuIndex Target CPU Index
  1311. @param[in, out] ProcArguments The parameter to pass to the procedure
  1312. @retval EFI_INVALID_PARAMETER CpuNumber not valid
  1313. @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
  1314. @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
  1315. @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
  1316. @retval EFI_SUCCESS The procedure has been successfully scheduled
  1317. **/
  1318. EFI_STATUS
  1319. EFIAPI
  1320. SmmBlockingStartupThisAp (
  1321. IN EFI_AP_PROCEDURE Procedure,
  1322. IN UINTN CpuIndex,
  1323. IN OUT VOID *ProcArguments OPTIONAL
  1324. )
  1325. {
  1326. PROCEDURE_WRAPPER Wrapper;
  1327. Wrapper.Procedure = Procedure;
  1328. Wrapper.ProcedureArgument = ProcArguments;
  1329. //
  1330. // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
  1331. //
  1332. return InternalSmmStartupThisAp (ProcedureWrapper, CpuIndex, &Wrapper, NULL, 0, NULL);
  1333. }
  1334. /**
  1335. Schedule a procedure to run on the specified CPU.
  1336. @param Procedure The address of the procedure to run
  1337. @param CpuIndex Target CPU Index
  1338. @param ProcArguments The parameter to pass to the procedure
  1339. @retval EFI_INVALID_PARAMETER CpuNumber not valid
  1340. @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
  1341. @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
  1342. @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
  1343. @retval EFI_SUCCESS The procedure has been successfully scheduled
  1344. **/
  1345. EFI_STATUS
  1346. EFIAPI
  1347. SmmStartupThisAp (
  1348. IN EFI_AP_PROCEDURE Procedure,
  1349. IN UINTN CpuIndex,
  1350. IN OUT VOID *ProcArguments OPTIONAL
  1351. )
  1352. {
  1353. gSmmCpuPrivate->ApWrapperFunc[CpuIndex].Procedure = Procedure;
  1354. gSmmCpuPrivate->ApWrapperFunc[CpuIndex].ProcedureArgument = ProcArguments;
  1355. //
  1356. // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
  1357. //
  1358. return InternalSmmStartupThisAp (
  1359. ProcedureWrapper,
  1360. CpuIndex,
  1361. &gSmmCpuPrivate->ApWrapperFunc[CpuIndex],
  1362. FeaturePcdGet (PcdCpuSmmBlockStartupThisAp) ? NULL : &mSmmStartupThisApToken,
  1363. 0,
  1364. NULL
  1365. );
  1366. }
  1367. /**
  1368. This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
  1369. They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
  1370. NOTE: It might not be appreciated in runtime since it might
  1371. conflict with OS debugging facilities. Turn them off in RELEASE.
  1372. @param CpuIndex CPU Index
  1373. **/
  1374. VOID
  1375. EFIAPI
  1376. CpuSmmDebugEntry (
  1377. IN UINTN CpuIndex
  1378. )
  1379. {
  1380. SMRAM_SAVE_STATE_MAP *CpuSaveState;
  1381. if (FeaturePcdGet (PcdCpuSmmDebug)) {
  1382. ASSERT (CpuIndex < mMaxNumberOfCpus);
  1383. CpuSaveState = (SMRAM_SAVE_STATE_MAP *)gSmmCpuPrivate->CpuSaveState[CpuIndex];
  1384. if (mSmmSaveStateRegisterLma == EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT) {
  1385. AsmWriteDr6 (CpuSaveState->x86._DR6);
  1386. AsmWriteDr7 (CpuSaveState->x86._DR7);
  1387. } else {
  1388. AsmWriteDr6 ((UINTN)CpuSaveState->x64._DR6);
  1389. AsmWriteDr7 ((UINTN)CpuSaveState->x64._DR7);
  1390. }
  1391. }
  1392. }
  1393. /**
  1394. This function restores DR6 & DR7 to SMM save state.
  1395. NOTE: It might not be appreciated in runtime since it might
  1396. conflict with OS debugging facilities. Turn them off in RELEASE.
  1397. @param CpuIndex CPU Index
  1398. **/
  1399. VOID
  1400. EFIAPI
  1401. CpuSmmDebugExit (
  1402. IN UINTN CpuIndex
  1403. )
  1404. {
  1405. SMRAM_SAVE_STATE_MAP *CpuSaveState;
  1406. if (FeaturePcdGet (PcdCpuSmmDebug)) {
  1407. ASSERT (CpuIndex < mMaxNumberOfCpus);
  1408. CpuSaveState = (SMRAM_SAVE_STATE_MAP *)gSmmCpuPrivate->CpuSaveState[CpuIndex];
  1409. if (mSmmSaveStateRegisterLma == EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT) {
  1410. CpuSaveState->x86._DR7 = (UINT32)AsmReadDr7 ();
  1411. CpuSaveState->x86._DR6 = (UINT32)AsmReadDr6 ();
  1412. } else {
  1413. CpuSaveState->x64._DR7 = AsmReadDr7 ();
  1414. CpuSaveState->x64._DR6 = AsmReadDr6 ();
  1415. }
  1416. }
  1417. }
  1418. /**
  1419. C function for SMI entry, each processor comes here upon SMI trigger.
  1420. @param CpuIndex CPU Index
  1421. **/
  1422. VOID
  1423. EFIAPI
  1424. SmiRendezvous (
  1425. IN UINTN CpuIndex
  1426. )
  1427. {
  1428. EFI_STATUS Status;
  1429. BOOLEAN ValidSmi;
  1430. BOOLEAN IsBsp;
  1431. BOOLEAN BspInProgress;
  1432. UINTN Index;
  1433. UINTN Cr2;
  1434. ASSERT (CpuIndex < mMaxNumberOfCpus);
  1435. //
  1436. // Save Cr2 because Page Fault exception in SMM may override its value,
  1437. // when using on-demand paging for above 4G memory.
  1438. //
  1439. Cr2 = 0;
  1440. SaveCr2 (&Cr2);
  1441. //
  1442. // Call the user register Startup function first.
  1443. //
  1444. if (mSmmMpSyncData->StartupProcedure != NULL) {
  1445. mSmmMpSyncData->StartupProcedure (mSmmMpSyncData->StartupProcArgs);
  1446. }
  1447. //
  1448. // Perform CPU specific entry hooks
  1449. //
  1450. SmmCpuFeaturesRendezvousEntry (CpuIndex);
  1451. //
  1452. // Determine if this is a valid SMI
  1453. //
  1454. ValidSmi = PlatformValidSmi ();
  1455. //
  1456. // Determine if BSP has been already in progress. Note this must be checked after
  1457. // ValidSmi because BSP may clear a valid SMI source after checking in.
  1458. //
  1459. BspInProgress = *mSmmMpSyncData->InsideSmm;
  1460. if (!BspInProgress && !ValidSmi) {
  1461. //
  1462. // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not
  1463. // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI
  1464. // status had been cleared by BSP and an existing SMI run has almost ended. (Note
  1465. // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there
  1466. // is nothing we need to do.
  1467. //
  1468. goto Exit;
  1469. } else {
  1470. //
  1471. // Signal presence of this processor
  1472. // mSmmMpSyncData->Counter is increased here!
  1473. // "ReleaseSemaphore (mSmmMpSyncData->Counter) == 0" means BSP has already ended the synchronization.
  1474. //
  1475. if (ReleaseSemaphore (mSmmMpSyncData->Counter) == 0) {
  1476. //
  1477. // BSP has already ended the synchronization, so QUIT!!!
  1478. // Existing AP is too late now to enter SMI since BSP has already ended the synchronization!!!
  1479. //
  1480. //
  1481. // Wait for BSP's signal to finish SMI
  1482. //
  1483. while (*mSmmMpSyncData->AllCpusInSync) {
  1484. CpuPause ();
  1485. }
  1486. goto Exit;
  1487. } else {
  1488. //
  1489. // The BUSY lock is initialized to Released state.
  1490. // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.
  1491. // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
  1492. // after AP's present flag is detected.
  1493. //
  1494. InitializeSpinLock (mSmmMpSyncData->CpuData[CpuIndex].Busy);
  1495. }
  1496. if (FeaturePcdGet (PcdCpuSmmProfileEnable)) {
  1497. ActivateSmmProfile (CpuIndex);
  1498. }
  1499. if (BspInProgress) {
  1500. //
  1501. // BSP has been elected. Follow AP path, regardless of ValidSmi flag
  1502. // as BSP may have cleared the SMI status
  1503. //
  1504. APHandler (CpuIndex, ValidSmi, mSmmMpSyncData->EffectiveSyncMode);
  1505. } else {
  1506. //
  1507. // We have a valid SMI
  1508. //
  1509. //
  1510. // Elect BSP
  1511. //
  1512. IsBsp = FALSE;
  1513. if (FeaturePcdGet (PcdCpuSmmEnableBspElection)) {
  1514. if (!mSmmMpSyncData->SwitchBsp || mSmmMpSyncData->CandidateBsp[CpuIndex]) {
  1515. //
  1516. // Call platform hook to do BSP election
  1517. //
  1518. Status = PlatformSmmBspElection (&IsBsp);
  1519. if (EFI_SUCCESS == Status) {
  1520. //
  1521. // Platform hook determines successfully
  1522. //
  1523. if (IsBsp) {
  1524. mSmmMpSyncData->BspIndex = (UINT32)CpuIndex;
  1525. }
  1526. } else {
  1527. //
  1528. // Platform hook fails to determine, use default BSP election method
  1529. //
  1530. InterlockedCompareExchange32 (
  1531. (UINT32 *)&mSmmMpSyncData->BspIndex,
  1532. (UINT32)-1,
  1533. (UINT32)CpuIndex
  1534. );
  1535. }
  1536. }
  1537. }
  1538. //
  1539. // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP
  1540. //
  1541. if (mSmmMpSyncData->BspIndex == CpuIndex) {
  1542. //
  1543. // Clear last request for SwitchBsp.
  1544. //
  1545. if (mSmmMpSyncData->SwitchBsp) {
  1546. mSmmMpSyncData->SwitchBsp = FALSE;
  1547. for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
  1548. mSmmMpSyncData->CandidateBsp[Index] = FALSE;
  1549. }
  1550. }
  1551. if (FeaturePcdGet (PcdCpuSmmProfileEnable)) {
  1552. SmmProfileRecordSmiNum ();
  1553. }
  1554. //
  1555. // BSP Handler is always called with a ValidSmi == TRUE
  1556. //
  1557. BSPHandler (CpuIndex, mSmmMpSyncData->EffectiveSyncMode);
  1558. } else {
  1559. APHandler (CpuIndex, ValidSmi, mSmmMpSyncData->EffectiveSyncMode);
  1560. }
  1561. }
  1562. ASSERT (*mSmmMpSyncData->CpuData[CpuIndex].Run == 0);
  1563. //
  1564. // Wait for BSP's signal to exit SMI
  1565. //
  1566. while (*mSmmMpSyncData->AllCpusInSync) {
  1567. CpuPause ();
  1568. }
  1569. }
  1570. Exit:
  1571. SmmCpuFeaturesRendezvousExit (CpuIndex);
  1572. //
  1573. // Restore Cr2
  1574. //
  1575. RestoreCr2 (Cr2);
  1576. }
  1577. /**
  1578. Initialize PackageBsp Info. Processor specified by mPackageFirstThreadIndex[PackageIndex]
  1579. will do the package-scope register programming. Set default CpuIndex to (UINT32)-1, which
  1580. means not specified yet.
  1581. **/
  1582. VOID
  1583. InitPackageFirstThreadIndexInfo (
  1584. VOID
  1585. )
  1586. {
  1587. UINT32 Index;
  1588. UINT32 PackageId;
  1589. UINT32 PackageCount;
  1590. PackageId = 0;
  1591. PackageCount = 0;
  1592. //
  1593. // Count the number of package, set to max PackageId + 1
  1594. //
  1595. for (Index = 0; Index < mNumberOfCpus; Index++) {
  1596. if (PackageId < gSmmCpuPrivate->ProcessorInfo[Index].Location.Package) {
  1597. PackageId = gSmmCpuPrivate->ProcessorInfo[Index].Location.Package;
  1598. }
  1599. }
  1600. PackageCount = PackageId + 1;
  1601. mPackageFirstThreadIndex = (UINT32 *)AllocatePool (sizeof (UINT32) * PackageCount);
  1602. ASSERT (mPackageFirstThreadIndex != NULL);
  1603. if (mPackageFirstThreadIndex == NULL) {
  1604. return;
  1605. }
  1606. //
  1607. // Set default CpuIndex to (UINT32)-1, which means not specified yet.
  1608. //
  1609. SetMem32 (mPackageFirstThreadIndex, sizeof (UINT32) * PackageCount, (UINT32)-1);
  1610. }
  1611. /**
  1612. Allocate buffer for SpinLock and Wrapper function buffer.
  1613. **/
  1614. VOID
  1615. InitializeDataForMmMp (
  1616. VOID
  1617. )
  1618. {
  1619. gSmmCpuPrivate->ApWrapperFunc = AllocatePool (sizeof (PROCEDURE_WRAPPER) * gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus);
  1620. ASSERT (gSmmCpuPrivate->ApWrapperFunc != NULL);
  1621. InitializeListHead (&gSmmCpuPrivate->TokenList);
  1622. gSmmCpuPrivate->FirstFreeToken = AllocateTokenBuffer ();
  1623. }
  1624. /**
  1625. Allocate buffer for all semaphores and spin locks.
  1626. **/
  1627. VOID
  1628. InitializeSmmCpuSemaphores (
  1629. VOID
  1630. )
  1631. {
  1632. UINTN ProcessorCount;
  1633. UINTN TotalSize;
  1634. UINTN GlobalSemaphoresSize;
  1635. UINTN CpuSemaphoresSize;
  1636. UINTN SemaphoreSize;
  1637. UINTN Pages;
  1638. UINTN *SemaphoreBlock;
  1639. UINTN SemaphoreAddr;
  1640. SemaphoreSize = GetSpinLockProperties ();
  1641. ProcessorCount = gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
  1642. GlobalSemaphoresSize = (sizeof (SMM_CPU_SEMAPHORE_GLOBAL) / sizeof (VOID *)) * SemaphoreSize;
  1643. CpuSemaphoresSize = (sizeof (SMM_CPU_SEMAPHORE_CPU) / sizeof (VOID *)) * ProcessorCount * SemaphoreSize;
  1644. TotalSize = GlobalSemaphoresSize + CpuSemaphoresSize;
  1645. DEBUG ((DEBUG_INFO, "One Semaphore Size = 0x%x\n", SemaphoreSize));
  1646. DEBUG ((DEBUG_INFO, "Total Semaphores Size = 0x%x\n", TotalSize));
  1647. Pages = EFI_SIZE_TO_PAGES (TotalSize);
  1648. SemaphoreBlock = AllocatePages (Pages);
  1649. ASSERT (SemaphoreBlock != NULL);
  1650. ZeroMem (SemaphoreBlock, TotalSize);
  1651. SemaphoreAddr = (UINTN)SemaphoreBlock;
  1652. mSmmCpuSemaphores.SemaphoreGlobal.Counter = (UINT32 *)SemaphoreAddr;
  1653. SemaphoreAddr += SemaphoreSize;
  1654. mSmmCpuSemaphores.SemaphoreGlobal.InsideSmm = (BOOLEAN *)SemaphoreAddr;
  1655. SemaphoreAddr += SemaphoreSize;
  1656. mSmmCpuSemaphores.SemaphoreGlobal.AllCpusInSync = (BOOLEAN *)SemaphoreAddr;
  1657. SemaphoreAddr += SemaphoreSize;
  1658. mSmmCpuSemaphores.SemaphoreGlobal.PFLock = (SPIN_LOCK *)SemaphoreAddr;
  1659. SemaphoreAddr += SemaphoreSize;
  1660. mSmmCpuSemaphores.SemaphoreGlobal.CodeAccessCheckLock
  1661. = (SPIN_LOCK *)SemaphoreAddr;
  1662. SemaphoreAddr += SemaphoreSize;
  1663. SemaphoreAddr = (UINTN)SemaphoreBlock + GlobalSemaphoresSize;
  1664. mSmmCpuSemaphores.SemaphoreCpu.Busy = (SPIN_LOCK *)SemaphoreAddr;
  1665. SemaphoreAddr += ProcessorCount * SemaphoreSize;
  1666. mSmmCpuSemaphores.SemaphoreCpu.Run = (UINT32 *)SemaphoreAddr;
  1667. SemaphoreAddr += ProcessorCount * SemaphoreSize;
  1668. mSmmCpuSemaphores.SemaphoreCpu.Present = (BOOLEAN *)SemaphoreAddr;
  1669. mPFLock = mSmmCpuSemaphores.SemaphoreGlobal.PFLock;
  1670. mConfigSmmCodeAccessCheckLock = mSmmCpuSemaphores.SemaphoreGlobal.CodeAccessCheckLock;
  1671. mSemaphoreSize = SemaphoreSize;
  1672. }
  1673. /**
  1674. Initialize un-cacheable data.
  1675. **/
  1676. VOID
  1677. EFIAPI
  1678. InitializeMpSyncData (
  1679. VOID
  1680. )
  1681. {
  1682. UINTN CpuIndex;
  1683. if (mSmmMpSyncData != NULL) {
  1684. //
  1685. // mSmmMpSyncDataSize includes one structure of SMM_DISPATCHER_MP_SYNC_DATA, one
  1686. // CpuData array of SMM_CPU_DATA_BLOCK and one CandidateBsp array of BOOLEAN.
  1687. //
  1688. ZeroMem (mSmmMpSyncData, mSmmMpSyncDataSize);
  1689. mSmmMpSyncData->CpuData = (SMM_CPU_DATA_BLOCK *)((UINT8 *)mSmmMpSyncData + sizeof (SMM_DISPATCHER_MP_SYNC_DATA));
  1690. mSmmMpSyncData->CandidateBsp = (BOOLEAN *)(mSmmMpSyncData->CpuData + gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus);
  1691. if (FeaturePcdGet (PcdCpuSmmEnableBspElection)) {
  1692. //
  1693. // Enable BSP election by setting BspIndex to -1
  1694. //
  1695. mSmmMpSyncData->BspIndex = (UINT32)-1;
  1696. }
  1697. mSmmMpSyncData->EffectiveSyncMode = mCpuSmmSyncMode;
  1698. mSmmMpSyncData->Counter = mSmmCpuSemaphores.SemaphoreGlobal.Counter;
  1699. mSmmMpSyncData->InsideSmm = mSmmCpuSemaphores.SemaphoreGlobal.InsideSmm;
  1700. mSmmMpSyncData->AllCpusInSync = mSmmCpuSemaphores.SemaphoreGlobal.AllCpusInSync;
  1701. ASSERT (
  1702. mSmmMpSyncData->Counter != NULL && mSmmMpSyncData->InsideSmm != NULL &&
  1703. mSmmMpSyncData->AllCpusInSync != NULL
  1704. );
  1705. *mSmmMpSyncData->Counter = 0;
  1706. *mSmmMpSyncData->InsideSmm = FALSE;
  1707. *mSmmMpSyncData->AllCpusInSync = FALSE;
  1708. mSmmMpSyncData->AllApArrivedWithException = FALSE;
  1709. for (CpuIndex = 0; CpuIndex < gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus; CpuIndex++) {
  1710. mSmmMpSyncData->CpuData[CpuIndex].Busy =
  1711. (SPIN_LOCK *)((UINTN)mSmmCpuSemaphores.SemaphoreCpu.Busy + mSemaphoreSize * CpuIndex);
  1712. mSmmMpSyncData->CpuData[CpuIndex].Run =
  1713. (UINT32 *)((UINTN)mSmmCpuSemaphores.SemaphoreCpu.Run + mSemaphoreSize * CpuIndex);
  1714. mSmmMpSyncData->CpuData[CpuIndex].Present =
  1715. (BOOLEAN *)((UINTN)mSmmCpuSemaphores.SemaphoreCpu.Present + mSemaphoreSize * CpuIndex);
  1716. *(mSmmMpSyncData->CpuData[CpuIndex].Busy) = 0;
  1717. *(mSmmMpSyncData->CpuData[CpuIndex].Run) = 0;
  1718. *(mSmmMpSyncData->CpuData[CpuIndex].Present) = FALSE;
  1719. }
  1720. }
  1721. }
  1722. /**
  1723. Initialize global data for MP synchronization.
  1724. @param Stacks Base address of SMI stack buffer for all processors.
  1725. @param StackSize Stack size for each processor in SMM.
  1726. @param ShadowStackSize Shadow Stack size for each processor in SMM.
  1727. **/
  1728. UINT32
  1729. InitializeMpServiceData (
  1730. IN VOID *Stacks,
  1731. IN UINTN StackSize,
  1732. IN UINTN ShadowStackSize
  1733. )
  1734. {
  1735. UINT32 Cr3;
  1736. UINTN Index;
  1737. UINT8 *GdtTssTables;
  1738. UINTN GdtTableStepSize;
  1739. CPUID_VERSION_INFO_EDX RegEdx;
  1740. UINT32 MaxExtendedFunction;
  1741. CPUID_VIR_PHY_ADDRESS_SIZE_EAX VirPhyAddressSize;
  1742. //
  1743. // Determine if this CPU supports machine check
  1744. //
  1745. AsmCpuid (CPUID_VERSION_INFO, NULL, NULL, NULL, &RegEdx.Uint32);
  1746. mMachineCheckSupported = (BOOLEAN)(RegEdx.Bits.MCA == 1);
  1747. //
  1748. // Allocate memory for all locks and semaphores
  1749. //
  1750. InitializeSmmCpuSemaphores ();
  1751. //
  1752. // Initialize mSmmMpSyncData
  1753. //
  1754. mSmmMpSyncDataSize = sizeof (SMM_DISPATCHER_MP_SYNC_DATA) +
  1755. (sizeof (SMM_CPU_DATA_BLOCK) + sizeof (BOOLEAN)) * gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus;
  1756. mSmmMpSyncData = (SMM_DISPATCHER_MP_SYNC_DATA *)AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize));
  1757. ASSERT (mSmmMpSyncData != NULL);
  1758. mCpuSmmSyncMode = (SMM_CPU_SYNC_MODE)PcdGet8 (PcdCpuSmmSyncMode);
  1759. InitializeMpSyncData ();
  1760. //
  1761. // Initialize physical address mask
  1762. // NOTE: Physical memory above virtual address limit is not supported !!!
  1763. //
  1764. AsmCpuid (CPUID_EXTENDED_FUNCTION, &MaxExtendedFunction, NULL, NULL, NULL);
  1765. if (MaxExtendedFunction >= CPUID_VIR_PHY_ADDRESS_SIZE) {
  1766. AsmCpuid (CPUID_VIR_PHY_ADDRESS_SIZE, &VirPhyAddressSize.Uint32, NULL, NULL, NULL);
  1767. } else {
  1768. VirPhyAddressSize.Bits.PhysicalAddressBits = 36;
  1769. }
  1770. gPhyMask = LShiftU64 (1, VirPhyAddressSize.Bits.PhysicalAddressBits) - 1;
  1771. //
  1772. // Clear the low 12 bits
  1773. //
  1774. gPhyMask &= 0xfffffffffffff000ULL;
  1775. //
  1776. // Create page tables
  1777. //
  1778. Cr3 = SmmInitPageTable ();
  1779. GdtTssTables = InitGdt (Cr3, &GdtTableStepSize);
  1780. //
  1781. // Install SMI handler for each CPU
  1782. //
  1783. for (Index = 0; Index < mMaxNumberOfCpus; Index++) {
  1784. InstallSmiHandler (
  1785. Index,
  1786. (UINT32)mCpuHotPlugData.SmBase[Index],
  1787. (VOID *)((UINTN)Stacks + (StackSize + ShadowStackSize) * Index),
  1788. StackSize,
  1789. (UINTN)(GdtTssTables + GdtTableStepSize * Index),
  1790. gcSmiGdtr.Limit + 1,
  1791. gcSmiIdtr.Base,
  1792. gcSmiIdtr.Limit + 1,
  1793. Cr3
  1794. );
  1795. }
  1796. //
  1797. // Record current MTRR settings
  1798. //
  1799. ZeroMem (&gSmiMtrrs, sizeof (gSmiMtrrs));
  1800. MtrrGetAllMtrrs (&gSmiMtrrs);
  1801. return Cr3;
  1802. }
  1803. /**
  1804. Register the SMM Foundation entry point.
  1805. @param This Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance
  1806. @param SmmEntryPoint SMM Foundation EntryPoint
  1807. @retval EFI_SUCCESS Successfully to register SMM foundation entry point
  1808. **/
  1809. EFI_STATUS
  1810. EFIAPI
  1811. RegisterSmmEntry (
  1812. IN CONST EFI_SMM_CONFIGURATION_PROTOCOL *This,
  1813. IN EFI_SMM_ENTRY_POINT SmmEntryPoint
  1814. )
  1815. {
  1816. //
  1817. // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.
  1818. //
  1819. gSmmCpuPrivate->SmmCoreEntry = SmmEntryPoint;
  1820. return EFI_SUCCESS;
  1821. }
  1822. /**
  1823. Register the SMM Foundation entry point.
  1824. @param[in] Procedure A pointer to the code stream to be run on the designated target AP
  1825. of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
  1826. with the related definitions of
  1827. EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
  1828. If caller may pass a value of NULL to deregister any existing
  1829. startup procedure.
  1830. @param[in,out] ProcedureArguments Allows the caller to pass a list of parameters to the code that is
  1831. run by the AP. It is an optional common mailbox between APs and
  1832. the caller to share information
  1833. @retval EFI_SUCCESS The Procedure has been set successfully.
  1834. @retval EFI_INVALID_PARAMETER The Procedure is NULL but ProcedureArguments not NULL.
  1835. **/
  1836. EFI_STATUS
  1837. RegisterStartupProcedure (
  1838. IN EFI_AP_PROCEDURE Procedure,
  1839. IN OUT VOID *ProcedureArguments OPTIONAL
  1840. )
  1841. {
  1842. if ((Procedure == NULL) && (ProcedureArguments != NULL)) {
  1843. return EFI_INVALID_PARAMETER;
  1844. }
  1845. if (mSmmMpSyncData == NULL) {
  1846. return EFI_NOT_READY;
  1847. }
  1848. mSmmMpSyncData->StartupProcedure = Procedure;
  1849. mSmmMpSyncData->StartupProcArgs = ProcedureArguments;
  1850. return EFI_SUCCESS;
  1851. }