mp_init.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * Copyright (C) 2015 Google, Inc
  4. *
  5. * Based on code from the coreboot file of the same name
  6. */
  7. #include <common.h>
  8. #include <cpu.h>
  9. #include <dm.h>
  10. #include <errno.h>
  11. #include <log.h>
  12. #include <malloc.h>
  13. #include <qfw.h>
  14. #include <asm/atomic.h>
  15. #include <asm/cpu.h>
  16. #include <asm/global_data.h>
  17. #include <asm/interrupt.h>
  18. #include <asm/io.h>
  19. #include <asm/lapic.h>
  20. #include <asm/microcode.h>
  21. #include <asm/mp.h>
  22. #include <asm/msr.h>
  23. #include <asm/mtrr.h>
  24. #include <asm/processor.h>
  25. #include <asm/sipi.h>
  26. #include <dm/device-internal.h>
  27. #include <dm/uclass-internal.h>
  28. #include <dm/lists.h>
  29. #include <dm/root.h>
  30. #include <linux/delay.h>
  31. #include <linux/linkage.h>
  32. DECLARE_GLOBAL_DATA_PTR;
  33. /*
  34. * Setting up multiprocessing
  35. *
  36. * See https://www.intel.com/content/www/us/en/intelligent-systems/intel-boot-loader-development-kit/minimal-intel-architecture-boot-loader-paper.html
  37. *
  38. * Note that this file refers to the boot CPU (the one U-Boot is running on) as
  39. * the BSP (BootStrap Processor) and the others as APs (Application Processors).
  40. *
  41. * This module works by loading some setup code into RAM at AP_DEFAULT_BASE and
  42. * telling each AP to execute it. The code that each AP runs is in
  43. * sipi_vector.S (see ap_start16) which includes a struct sipi_params at the
  44. * end of it. Those parameters are set up by the C code.
  45. * Setting up is handled by load_sipi_vector(). It inits the common block of
  46. * parameters (sipi_params) which tell the APs what to do. This block includes
  47. * microcode and the MTTRs (Memory-Type-Range Registers) from the main CPU.
  48. * There is also an ap_count which each AP increments as it starts up, so the
  49. * BSP can tell how many checked in.
  50. *
  51. * The APs are started with a SIPI (Startup Inter-Processor Interrupt) which
  52. * tells an AP to start executing at a particular address, in this case
  53. * AP_DEFAULT_BASE which contains the code copied from ap_start16. This protocol
  54. * is handled by start_aps().
  55. *
  56. * After being started, each AP runs the code in ap_start16, switches to 32-bit
  57. * mode, runs the code at ap_start, then jumps to c_handler which is ap_init().
  58. * This runs a very simple 'flight plan' described in mp_steps(). This sets up
  59. * the CPU and waits for further instructions by looking at its entry in
  60. * ap_callbacks[]. Note that the flight plan is only actually run for each CPU
  61. * in bsp_do_flight_plan(): once the BSP completes each flight record, it sets
  62. * mp_flight_record->barrier to 1 to allow the APs to executed the record one
  63. * by one.
  64. *
  65. * CPUS are numbered sequentially from 0 using the device tree:
  66. *
  67. * cpus {
  68. * u-boot,dm-pre-reloc;
  69. * #address-cells = <1>;
  70. * #size-cells = <0>;
  71. *
  72. * cpu@0 {
  73. * u-boot,dm-pre-reloc;
  74. * device_type = "cpu";
  75. * compatible = "intel,apl-cpu";
  76. * reg = <0>;
  77. * intel,apic-id = <0>;
  78. * };
  79. *
  80. * cpu@1 {
  81. * device_type = "cpu";
  82. * compatible = "intel,apl-cpu";
  83. * reg = <1>;
  84. * intel,apic-id = <2>;
  85. * };
  86. *
  87. * Here the 'reg' property is the CPU number and then is placed in dev_seq(cpu)
  88. * so that we can index into ap_callbacks[] using that. The APIC ID is different
  89. * and may not be sequential (it typically is if hyperthreading is supported).
  90. *
  91. * Once APs are inited they wait in ap_wait_for_instruction() for instructions.
  92. * Instructions come in the form of a function to run. This logic is in
  93. * mp_run_on_cpus() which supports running on any one AP, all APs, just the BSP
  94. * or all CPUs. The BSP logic is handled directly in mp_run_on_cpus(), by
  95. * calling the function. For the APs, callback information is stored in a
  96. * single, common struct mp_callback and a pointer to this is written to each
  97. * AP's slot in ap_callbacks[] by run_ap_work(). All APs get the message even
  98. * if it is only for one of them. When an AP notices a message it checks whether
  99. * it should call the function (see check in ap_wait_for_instruction()) and then
  100. * does so if needed. After that it sets its slot to NULL to indicate it is
  101. * done.
  102. *
  103. * While U-Boot is running it can use mp_run_on_cpus() to run code on the APs.
  104. * An example of this is the 'mtrr' command which allows reading and changing
  105. * the MTRRs on all CPUs.
  106. *
  107. * Before U-Boot exits it calls mp_park_aps() which tells all CPUs to halt by
  108. * executing a 'hlt' instruction. That allows them to be used by Linux when it
  109. * starts up.
  110. */
  111. /* This also needs to match the sipi.S assembly code for saved MSR encoding */
  112. struct __packed saved_msr {
  113. uint32_t index;
  114. uint32_t lo;
  115. uint32_t hi;
  116. };
  117. /**
  118. * struct mp_flight_plan - Holds the flight plan
  119. *
  120. * @num_records: Number of flight records
  121. * @records: Pointer to each record
  122. */
  123. struct mp_flight_plan {
  124. int num_records;
  125. struct mp_flight_record *records;
  126. };
  127. /**
  128. * struct mp_callback - Callback information for APs
  129. *
  130. * @func: Function to run
  131. * @arg: Argument to pass to the function
  132. * @logical_cpu_number: Either a CPU number (i.e. dev_seq(cpu) or a special
  133. * value like MP_SELECT_BSP. It tells the AP whether it should process this
  134. * callback
  135. */
  136. struct mp_callback {
  137. mp_run_func func;
  138. void *arg;
  139. int logical_cpu_number;
  140. };
  141. /* Stores the flight plan so that APs can find it */
  142. static struct mp_flight_plan mp_info;
  143. /*
  144. * ap_callbacks - Callback mailbox array
  145. *
  146. * Array of callback, one entry for each available CPU, indexed by the CPU
  147. * number, which is dev_seq(cpu). The entry for the main CPU is never used.
  148. * When this is NULL, there is no pending work for the CPU to run. When
  149. * non-NULL it points to the mp_callback structure. This is shared between all
  150. * CPUs, so should only be written by the main CPU.
  151. */
  152. static struct mp_callback **ap_callbacks;
  153. static inline void barrier_wait(atomic_t *b)
  154. {
  155. while (atomic_read(b) == 0)
  156. asm("pause");
  157. mfence();
  158. }
  159. static inline void release_barrier(atomic_t *b)
  160. {
  161. mfence();
  162. atomic_set(b, 1);
  163. }
  164. static inline void stop_this_cpu(void)
  165. {
  166. /* Called by an AP when it is ready to halt and wait for a new task */
  167. for (;;)
  168. cpu_hlt();
  169. }
  170. /* Returns 1 if timeout waiting for APs. 0 if target APs found */
  171. static int wait_for_aps(atomic_t *val, int target, int total_delay,
  172. int delay_step)
  173. {
  174. int timeout = 0;
  175. int delayed = 0;
  176. while (atomic_read(val) != target) {
  177. udelay(delay_step);
  178. delayed += delay_step;
  179. if (delayed >= total_delay) {
  180. timeout = 1;
  181. break;
  182. }
  183. }
  184. return timeout;
  185. }
  186. static void ap_do_flight_plan(struct udevice *cpu)
  187. {
  188. int i;
  189. for (i = 0; i < mp_info.num_records; i++) {
  190. struct mp_flight_record *rec = &mp_info.records[i];
  191. atomic_inc(&rec->cpus_entered);
  192. barrier_wait(&rec->barrier);
  193. if (rec->ap_call != NULL)
  194. rec->ap_call(cpu, rec->ap_arg);
  195. }
  196. }
  197. static int find_cpu_by_apic_id(int apic_id, struct udevice **devp)
  198. {
  199. struct udevice *dev;
  200. *devp = NULL;
  201. for (uclass_find_first_device(UCLASS_CPU, &dev);
  202. dev;
  203. uclass_find_next_device(&dev)) {
  204. struct cpu_plat *plat = dev_get_parent_plat(dev);
  205. if (plat->cpu_id == apic_id) {
  206. *devp = dev;
  207. return 0;
  208. }
  209. }
  210. return -ENOENT;
  211. }
  212. /*
  213. * By the time APs call ap_init() caching has been setup, and microcode has
  214. * been loaded
  215. */
  216. static void ap_init(unsigned int cpu_index)
  217. {
  218. struct udevice *dev;
  219. int apic_id;
  220. int ret;
  221. /* Ensure the local apic is enabled */
  222. enable_lapic();
  223. apic_id = lapicid();
  224. ret = find_cpu_by_apic_id(apic_id, &dev);
  225. if (ret) {
  226. debug("Unknown CPU apic_id %x\n", apic_id);
  227. goto done;
  228. }
  229. debug("AP: slot %d apic_id %x, dev %s\n", cpu_index, apic_id,
  230. dev ? dev->name : "(apic_id not found)");
  231. /*
  232. * Walk the flight plan, which only returns if CONFIG_SMP_AP_WORK is not
  233. * enabled
  234. */
  235. ap_do_flight_plan(dev);
  236. done:
  237. stop_this_cpu();
  238. }
  239. static const unsigned int fixed_mtrrs[NUM_FIXED_MTRRS] = {
  240. MTRR_FIX_64K_00000_MSR, MTRR_FIX_16K_80000_MSR, MTRR_FIX_16K_A0000_MSR,
  241. MTRR_FIX_4K_C0000_MSR, MTRR_FIX_4K_C8000_MSR, MTRR_FIX_4K_D0000_MSR,
  242. MTRR_FIX_4K_D8000_MSR, MTRR_FIX_4K_E0000_MSR, MTRR_FIX_4K_E8000_MSR,
  243. MTRR_FIX_4K_F0000_MSR, MTRR_FIX_4K_F8000_MSR,
  244. };
  245. static inline struct saved_msr *save_msr(int index, struct saved_msr *entry)
  246. {
  247. msr_t msr;
  248. msr = msr_read(index);
  249. entry->index = index;
  250. entry->lo = msr.lo;
  251. entry->hi = msr.hi;
  252. /* Return the next entry */
  253. entry++;
  254. return entry;
  255. }
  256. static int save_bsp_msrs(char *start, int size)
  257. {
  258. int msr_count;
  259. int num_var_mtrrs;
  260. struct saved_msr *msr_entry;
  261. int i;
  262. msr_t msr;
  263. /* Determine number of MTRRs need to be saved */
  264. msr = msr_read(MTRR_CAP_MSR);
  265. num_var_mtrrs = msr.lo & 0xff;
  266. /* 2 * num_var_mtrrs for base and mask. +1 for IA32_MTRR_DEF_TYPE */
  267. msr_count = 2 * num_var_mtrrs + NUM_FIXED_MTRRS + 1;
  268. if ((msr_count * sizeof(struct saved_msr)) > size) {
  269. printf("Cannot mirror all %d msrs\n", msr_count);
  270. return -ENOSPC;
  271. }
  272. msr_entry = (void *)start;
  273. for (i = 0; i < NUM_FIXED_MTRRS; i++)
  274. msr_entry = save_msr(fixed_mtrrs[i], msr_entry);
  275. for (i = 0; i < num_var_mtrrs; i++) {
  276. msr_entry = save_msr(MTRR_PHYS_BASE_MSR(i), msr_entry);
  277. msr_entry = save_msr(MTRR_PHYS_MASK_MSR(i), msr_entry);
  278. }
  279. msr_entry = save_msr(MTRR_DEF_TYPE_MSR, msr_entry);
  280. return msr_count;
  281. }
  282. static int load_sipi_vector(atomic_t **ap_countp, int num_cpus)
  283. {
  284. struct sipi_params_16bit *params16;
  285. struct sipi_params *params;
  286. static char msr_save[512];
  287. char *stack;
  288. ulong addr;
  289. int code_len;
  290. int size;
  291. int ret;
  292. /* Copy in the code */
  293. code_len = ap_start16_code_end - ap_start16;
  294. debug("Copying SIPI code to %x: %d bytes\n", AP_DEFAULT_BASE,
  295. code_len);
  296. memcpy((void *)AP_DEFAULT_BASE, ap_start16, code_len);
  297. addr = AP_DEFAULT_BASE + (ulong)sipi_params_16bit - (ulong)ap_start16;
  298. params16 = (struct sipi_params_16bit *)addr;
  299. params16->ap_start = (uint32_t)ap_start;
  300. params16->gdt = (uint32_t)gd->arch.gdt;
  301. params16->gdt_limit = X86_GDT_SIZE - 1;
  302. debug("gdt = %x, gdt_limit = %x\n", params16->gdt, params16->gdt_limit);
  303. params = (struct sipi_params *)sipi_params;
  304. debug("SIPI 32-bit params at %p\n", params);
  305. params->idt_ptr = (uint32_t)x86_get_idt();
  306. params->stack_size = CONFIG_AP_STACK_SIZE;
  307. size = params->stack_size * num_cpus;
  308. stack = memalign(4096, size);
  309. if (!stack)
  310. return -ENOMEM;
  311. params->stack_top = (u32)(stack + size);
  312. #if !defined(CONFIG_QEMU) && !defined(CONFIG_HAVE_FSP) && \
  313. !defined(CONFIG_INTEL_MID)
  314. params->microcode_ptr = ucode_base;
  315. debug("Microcode at %x\n", params->microcode_ptr);
  316. #endif
  317. params->msr_table_ptr = (u32)msr_save;
  318. ret = save_bsp_msrs(msr_save, sizeof(msr_save));
  319. if (ret < 0)
  320. return ret;
  321. params->msr_count = ret;
  322. params->c_handler = (uint32_t)&ap_init;
  323. *ap_countp = &params->ap_count;
  324. atomic_set(*ap_countp, 0);
  325. debug("SIPI vector is ready\n");
  326. return 0;
  327. }
  328. static int check_cpu_devices(int expected_cpus)
  329. {
  330. int i;
  331. for (i = 0; i < expected_cpus; i++) {
  332. struct udevice *dev;
  333. int ret;
  334. ret = uclass_find_device(UCLASS_CPU, i, &dev);
  335. if (ret) {
  336. debug("Cannot find CPU %d in device tree\n", i);
  337. return ret;
  338. }
  339. }
  340. return 0;
  341. }
  342. /* Returns 1 for timeout. 0 on success */
  343. static int apic_wait_timeout(int total_delay, const char *msg)
  344. {
  345. int total = 0;
  346. if (!(lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY))
  347. return 0;
  348. debug("Waiting for %s...", msg);
  349. while (lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY) {
  350. udelay(50);
  351. total += 50;
  352. if (total >= total_delay) {
  353. debug("timed out: aborting\n");
  354. return -ETIMEDOUT;
  355. }
  356. }
  357. debug("done\n");
  358. return 0;
  359. }
  360. /**
  361. * start_aps() - Start up the APs and count how many we find
  362. *
  363. * This is called on the boot processor to start up all the other processors
  364. * (here called APs).
  365. *
  366. * @num_aps: Number of APs we expect to find
  367. * @ap_count: Initially zero. Incremented by this function for each AP found
  368. * @return 0 if all APs were set up correctly or there are none to set up,
  369. * -ENOSPC if the SIPI vector is too high in memory,
  370. * -ETIMEDOUT if the ICR is busy or the second SIPI fails to complete
  371. * -EIO if not all APs check in correctly
  372. */
  373. static int start_aps(int num_aps, atomic_t *ap_count)
  374. {
  375. int sipi_vector;
  376. /* Max location is 4KiB below 1MiB */
  377. const int max_vector_loc = ((1 << 20) - (1 << 12)) >> 12;
  378. if (num_aps == 0)
  379. return 0;
  380. /* The vector is sent as a 4k aligned address in one byte */
  381. sipi_vector = AP_DEFAULT_BASE >> 12;
  382. if (sipi_vector > max_vector_loc) {
  383. printf("SIPI vector too large! 0x%08x\n",
  384. sipi_vector);
  385. return -ENOSPC;
  386. }
  387. debug("Attempting to start %d APs\n", num_aps);
  388. if (apic_wait_timeout(1000, "ICR not to be busy"))
  389. return -ETIMEDOUT;
  390. /* Send INIT IPI to all but self */
  391. lapic_write(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0));
  392. lapic_write(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT |
  393. LAPIC_DM_INIT);
  394. debug("Waiting for 10ms after sending INIT\n");
  395. mdelay(10);
  396. /* Send 1st SIPI */
  397. if (apic_wait_timeout(1000, "ICR not to be busy"))
  398. return -ETIMEDOUT;
  399. lapic_write(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0));
  400. lapic_write(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT |
  401. LAPIC_DM_STARTUP | sipi_vector);
  402. if (apic_wait_timeout(10000, "first SIPI to complete"))
  403. return -ETIMEDOUT;
  404. /* Wait for CPUs to check in up to 200 us */
  405. wait_for_aps(ap_count, num_aps, 200, 15);
  406. /* Send 2nd SIPI */
  407. if (apic_wait_timeout(1000, "ICR not to be busy"))
  408. return -ETIMEDOUT;
  409. lapic_write(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0));
  410. lapic_write(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT |
  411. LAPIC_DM_STARTUP | sipi_vector);
  412. if (apic_wait_timeout(10000, "second SIPI to complete"))
  413. return -ETIMEDOUT;
  414. /* Wait for CPUs to check in */
  415. if (wait_for_aps(ap_count, num_aps, 10000, 50)) {
  416. debug("Not all APs checked in: %d/%d\n",
  417. atomic_read(ap_count), num_aps);
  418. return -EIO;
  419. }
  420. return 0;
  421. }
  422. /**
  423. * bsp_do_flight_plan() - Do the flight plan on the BSP
  424. *
  425. * This runs the flight plan on the main CPU used to boot U-Boot
  426. *
  427. * @cpu: Device for the main CPU
  428. * @plan: Flight plan to run
  429. * @num_aps: Number of APs (CPUs other than the BSP)
  430. * @returns 0 on success, -ETIMEDOUT if an AP failed to come up
  431. */
  432. static int bsp_do_flight_plan(struct udevice *cpu, struct mp_flight_plan *plan,
  433. int num_aps)
  434. {
  435. int i;
  436. int ret = 0;
  437. const int timeout_us = 100000;
  438. const int step_us = 100;
  439. for (i = 0; i < plan->num_records; i++) {
  440. struct mp_flight_record *rec = &plan->records[i];
  441. /* Wait for APs if the record is not released */
  442. if (atomic_read(&rec->barrier) == 0) {
  443. /* Wait for the APs to check in */
  444. if (wait_for_aps(&rec->cpus_entered, num_aps,
  445. timeout_us, step_us)) {
  446. debug("MP record %d timeout\n", i);
  447. ret = -ETIMEDOUT;
  448. }
  449. }
  450. if (rec->bsp_call != NULL)
  451. rec->bsp_call(cpu, rec->bsp_arg);
  452. release_barrier(&rec->barrier);
  453. }
  454. return ret;
  455. }
  456. /**
  457. * get_bsp() - Get information about the bootstrap processor
  458. *
  459. * @devp: If non-NULL, returns CPU device corresponding to the BSP
  460. * @cpu_countp: If non-NULL, returns the total number of CPUs
  461. * @return CPU number of the BSP, or -ve on error. If multiprocessing is not
  462. * enabled, returns 0
  463. */
  464. static int get_bsp(struct udevice **devp, int *cpu_countp)
  465. {
  466. char processor_name[CPU_MAX_NAME_LEN];
  467. struct udevice *dev;
  468. int apic_id;
  469. int ret;
  470. cpu_get_name(processor_name);
  471. debug("CPU: %s\n", processor_name);
  472. apic_id = lapicid();
  473. ret = find_cpu_by_apic_id(apic_id, &dev);
  474. if (ret < 0) {
  475. printf("Cannot find boot CPU, APIC ID %d\n", apic_id);
  476. return ret;
  477. }
  478. ret = cpu_get_count(dev);
  479. if (ret < 0)
  480. return log_msg_ret("count", ret);
  481. if (devp)
  482. *devp = dev;
  483. if (cpu_countp)
  484. *cpu_countp = ret;
  485. return dev_seq(dev) >= 0 ? dev_seq(dev) : 0;
  486. }
  487. /**
  488. * read_callback() - Read the pointer in a callback slot
  489. *
  490. * This is called by APs to read their callback slot to see if there is a
  491. * pointer to new instructions
  492. *
  493. * @slot: Pointer to the AP's callback slot
  494. * @return value of that pointer
  495. */
  496. static struct mp_callback *read_callback(struct mp_callback **slot)
  497. {
  498. dmb();
  499. return *slot;
  500. }
  501. /**
  502. * store_callback() - Store a pointer to the callback slot
  503. *
  504. * This is called by APs to write NULL into the callback slot when they have
  505. * finished the work requested by the BSP.
  506. *
  507. * @slot: Pointer to the AP's callback slot
  508. * @val: Value to write (e.g. NULL)
  509. */
  510. static void store_callback(struct mp_callback **slot, struct mp_callback *val)
  511. {
  512. *slot = val;
  513. dmb();
  514. }
  515. /**
  516. * run_ap_work() - Run a callback on selected APs
  517. *
  518. * This writes @callback to all APs and waits for them all to acknowledge it,
  519. * Note that whether each AP actually calls the callback depends on the value
  520. * of logical_cpu_number (see struct mp_callback). The logical CPU number is
  521. * the CPU device's req->seq value.
  522. *
  523. * @callback: Callback information to pass to all APs
  524. * @bsp: CPU device for the BSP
  525. * @num_cpus: The number of CPUs in the system (= number of APs + 1)
  526. * @expire_ms: Timeout to wait for all APs to finish, in milliseconds, or 0 for
  527. * no timeout
  528. * @return 0 if OK, -ETIMEDOUT if one or more APs failed to respond in time
  529. */
  530. static int run_ap_work(struct mp_callback *callback, struct udevice *bsp,
  531. int num_cpus, uint expire_ms)
  532. {
  533. int cur_cpu = dev_seq(bsp);
  534. int num_aps = num_cpus - 1; /* number of non-BSPs to get this message */
  535. int cpus_accepted;
  536. ulong start;
  537. int i;
  538. if (!IS_ENABLED(CONFIG_SMP_AP_WORK)) {
  539. printf("APs already parked. CONFIG_SMP_AP_WORK not enabled\n");
  540. return -ENOTSUPP;
  541. }
  542. /* Signal to all the APs to run the func. */
  543. for (i = 0; i < num_cpus; i++) {
  544. if (cur_cpu != i)
  545. store_callback(&ap_callbacks[i], callback);
  546. }
  547. mfence();
  548. /* Wait for all the APs to signal back that call has been accepted. */
  549. start = get_timer(0);
  550. do {
  551. mdelay(1);
  552. cpus_accepted = 0;
  553. for (i = 0; i < num_cpus; i++) {
  554. if (cur_cpu == i)
  555. continue;
  556. if (!read_callback(&ap_callbacks[i]))
  557. cpus_accepted++;
  558. }
  559. if (expire_ms && get_timer(start) >= expire_ms) {
  560. log(UCLASS_CPU, LOGL_CRIT,
  561. "AP call expired; %d/%d CPUs accepted\n",
  562. cpus_accepted, num_aps);
  563. return -ETIMEDOUT;
  564. }
  565. } while (cpus_accepted != num_aps);
  566. /* Make sure we can see any data written by the APs */
  567. mfence();
  568. return 0;
  569. }
  570. /**
  571. * ap_wait_for_instruction() - Wait for and process requests from the main CPU
  572. *
  573. * This is called by APs (here, everything other than the main boot CPU) to
  574. * await instructions. They arrive in the form of a function call and argument,
  575. * which is then called. This uses a simple mailbox with atomic read/set
  576. *
  577. * @cpu: CPU that is waiting
  578. * @unused: Optional argument provided by struct mp_flight_record, not used here
  579. * @return Does not return
  580. */
  581. static int ap_wait_for_instruction(struct udevice *cpu, void *unused)
  582. {
  583. struct mp_callback lcb;
  584. struct mp_callback **per_cpu_slot;
  585. if (!IS_ENABLED(CONFIG_SMP_AP_WORK))
  586. return 0;
  587. per_cpu_slot = &ap_callbacks[dev_seq(cpu)];
  588. while (1) {
  589. struct mp_callback *cb = read_callback(per_cpu_slot);
  590. if (!cb) {
  591. asm ("pause");
  592. continue;
  593. }
  594. /* Copy to local variable before using the value */
  595. memcpy(&lcb, cb, sizeof(lcb));
  596. mfence();
  597. if (lcb.logical_cpu_number == MP_SELECT_ALL ||
  598. lcb.logical_cpu_number == MP_SELECT_APS ||
  599. dev_seq(cpu) == lcb.logical_cpu_number)
  600. lcb.func(lcb.arg);
  601. /* Indicate we are finished */
  602. store_callback(per_cpu_slot, NULL);
  603. }
  604. return 0;
  605. }
  606. static int mp_init_cpu(struct udevice *cpu, void *unused)
  607. {
  608. struct cpu_plat *plat = dev_get_parent_plat(cpu);
  609. plat->ucode_version = microcode_read_rev();
  610. plat->device_id = gd->arch.x86_device;
  611. return device_probe(cpu);
  612. }
  613. static struct mp_flight_record mp_steps[] = {
  614. MP_FR_BLOCK_APS(mp_init_cpu, NULL, mp_init_cpu, NULL),
  615. MP_FR_BLOCK_APS(ap_wait_for_instruction, NULL, NULL, NULL),
  616. };
  617. int mp_run_on_cpus(int cpu_select, mp_run_func func, void *arg)
  618. {
  619. struct mp_callback lcb = {
  620. .func = func,
  621. .arg = arg,
  622. .logical_cpu_number = cpu_select,
  623. };
  624. struct udevice *dev;
  625. int num_cpus;
  626. int ret;
  627. ret = get_bsp(&dev, &num_cpus);
  628. if (ret < 0)
  629. return log_msg_ret("bsp", ret);
  630. if (cpu_select == MP_SELECT_ALL || cpu_select == MP_SELECT_BSP ||
  631. cpu_select == ret) {
  632. /* Run on BSP first */
  633. func(arg);
  634. }
  635. if (!IS_ENABLED(CONFIG_SMP_AP_WORK) ||
  636. !(gd->flags & GD_FLG_SMP_READY)) {
  637. /* Allow use of this function on the BSP only */
  638. if (cpu_select == MP_SELECT_BSP || !cpu_select)
  639. return 0;
  640. return -ENOTSUPP;
  641. }
  642. /* Allow up to 1 second for all APs to finish */
  643. ret = run_ap_work(&lcb, dev, num_cpus, 1000 /* ms */);
  644. if (ret)
  645. return log_msg_ret("aps", ret);
  646. return 0;
  647. }
  648. static void park_this_cpu(void *unused)
  649. {
  650. stop_this_cpu();
  651. }
  652. int mp_park_aps(void)
  653. {
  654. int ret;
  655. ret = mp_run_on_cpus(MP_SELECT_APS, park_this_cpu, NULL);
  656. if (ret)
  657. return log_ret(ret);
  658. return 0;
  659. }
  660. int mp_first_cpu(int cpu_select)
  661. {
  662. struct udevice *dev;
  663. int num_cpus;
  664. int ret;
  665. /*
  666. * This assumes that CPUs are numbered from 0. This function tries to
  667. * avoid assuming the CPU 0 is the boot CPU
  668. */
  669. if (cpu_select == MP_SELECT_ALL)
  670. return 0; /* start with the first one */
  671. ret = get_bsp(&dev, &num_cpus);
  672. if (ret < 0)
  673. return log_msg_ret("bsp", ret);
  674. /* Return boot CPU if requested */
  675. if (cpu_select == MP_SELECT_BSP)
  676. return ret;
  677. /* Return something other than the boot CPU, if APs requested */
  678. if (cpu_select == MP_SELECT_APS && num_cpus > 1)
  679. return ret == 0 ? 1 : 0;
  680. /* Try to check for an invalid value */
  681. if (cpu_select < 0 || cpu_select >= num_cpus)
  682. return -EINVAL;
  683. return cpu_select; /* return the only selected one */
  684. }
  685. int mp_next_cpu(int cpu_select, int prev_cpu)
  686. {
  687. struct udevice *dev;
  688. int num_cpus;
  689. int ret;
  690. int bsp;
  691. /* If we selected the BSP or a particular single CPU, we are done */
  692. if (!IS_ENABLED(CONFIG_SMP_AP_WORK) || cpu_select == MP_SELECT_BSP ||
  693. cpu_select >= 0)
  694. return -EFBIG;
  695. /* Must be doing MP_SELECT_ALL or MP_SELECT_APS; return the next CPU */
  696. ret = get_bsp(&dev, &num_cpus);
  697. if (ret < 0)
  698. return log_msg_ret("bsp", ret);
  699. bsp = ret;
  700. /* Move to the next CPU */
  701. assert(prev_cpu >= 0);
  702. ret = prev_cpu + 1;
  703. /* Skip the BSP if needed */
  704. if (cpu_select == MP_SELECT_APS && ret == bsp)
  705. ret++;
  706. if (ret >= num_cpus)
  707. return -EFBIG;
  708. return ret;
  709. }
  710. int mp_init(void)
  711. {
  712. int num_aps, num_cpus;
  713. atomic_t *ap_count;
  714. struct udevice *cpu;
  715. int ret;
  716. if (IS_ENABLED(CONFIG_QFW)) {
  717. ret = qemu_cpu_fixup();
  718. if (ret)
  719. return ret;
  720. }
  721. ret = get_bsp(&cpu, &num_cpus);
  722. if (ret < 0) {
  723. debug("Cannot init boot CPU: err=%d\n", ret);
  724. return ret;
  725. }
  726. if (num_cpus < 2)
  727. debug("Warning: Only 1 CPU is detected\n");
  728. ret = check_cpu_devices(num_cpus);
  729. if (ret)
  730. log_warning("Warning: Device tree does not describe all CPUs. Extra ones will not be started correctly\n");
  731. ap_callbacks = calloc(num_cpus, sizeof(struct mp_callback *));
  732. if (!ap_callbacks)
  733. return -ENOMEM;
  734. /* Copy needed parameters so that APs have a reference to the plan */
  735. mp_info.num_records = ARRAY_SIZE(mp_steps);
  736. mp_info.records = mp_steps;
  737. /* Load the SIPI vector */
  738. ret = load_sipi_vector(&ap_count, num_cpus);
  739. if (ap_count == NULL)
  740. return -ENOENT;
  741. /*
  742. * Make sure SIPI data hits RAM so the APs that come up will see
  743. * the startup code even if the caches are disabled
  744. */
  745. wbinvd();
  746. /* Start the APs providing number of APs and the cpus_entered field */
  747. num_aps = num_cpus - 1;
  748. ret = start_aps(num_aps, ap_count);
  749. if (ret) {
  750. mdelay(1000);
  751. debug("%d/%d eventually checked in?\n", atomic_read(ap_count),
  752. num_aps);
  753. return ret;
  754. }
  755. /* Walk the flight plan for the BSP */
  756. ret = bsp_do_flight_plan(cpu, &mp_info, num_aps);
  757. if (ret) {
  758. debug("CPU init failed: err=%d\n", ret);
  759. return ret;
  760. }
  761. gd->flags |= GD_FLG_SMP_READY;
  762. return 0;
  763. }