sbi_init.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571
  1. /*
  2. * SPDX-License-Identifier: BSD-2-Clause
  3. *
  4. * Copyright (c) 2019 Western Digital Corporation or its affiliates.
  5. *
  6. * Authors:
  7. * Anup Patel <anup.patel@wdc.com>
  8. */
  9. #include <sbi/riscv_asm.h>
  10. #include <sbi/riscv_atomic.h>
  11. #include <sbi/riscv_barrier.h>
  12. #include <sbi/riscv_locks.h>
  13. #include <sbi/sbi_console.h>
  14. #include <sbi/sbi_domain.h>
  15. #include <sbi/sbi_ecall.h>
  16. #include <sbi/sbi_hart.h>
  17. #include <sbi/sbi_hartmask.h>
  18. #include <sbi/sbi_hsm.h>
  19. #include <sbi/sbi_ipi.h>
  20. #include <sbi/sbi_irqchip.h>
  21. #include <sbi/sbi_platform.h>
  22. #include <sbi/sbi_pmu.h>
  23. #include <sbi/sbi_system.h>
  24. #include <sbi/sbi_string.h>
  25. #include <sbi/sbi_timer.h>
  26. #include <sbi/sbi_tlb.h>
  27. #include <sbi/sbi_version.h>
  28. #define BANNER \
  29. " ____ _____ ____ _____\n" \
  30. " / __ \\ / ____| _ \\_ _|\n" \
  31. " | | | |_ __ ___ _ __ | (___ | |_) || |\n" \
  32. " | | | | '_ \\ / _ \\ '_ \\ \\___ \\| _ < | |\n" \
  33. " | |__| | |_) | __/ | | |____) | |_) || |_\n" \
  34. " \\____/| .__/ \\___|_| |_|_____/|____/_____|\n" \
  35. " | |\n" \
  36. " |_|\n\n"
  37. static void sbi_boot_print_banner(struct sbi_scratch *scratch)
  38. {
  39. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  40. return;
  41. #ifdef OPENSBI_VERSION_GIT
  42. sbi_printf("\nOpenSBI %s\n", OPENSBI_VERSION_GIT);
  43. #else
  44. sbi_printf("\nOpenSBI v%d.%d\n", OPENSBI_VERSION_MAJOR,
  45. OPENSBI_VERSION_MINOR);
  46. #endif
  47. #ifdef OPENSBI_BUILD_TIME_STAMP
  48. sbi_printf("Build time: %s\n", OPENSBI_BUILD_TIME_STAMP);
  49. #endif
  50. #ifdef OPENSBI_BUILD_COMPILER_VERSION
  51. sbi_printf("Build compiler: %s\n", OPENSBI_BUILD_COMPILER_VERSION);
  52. #endif
  53. sbi_printf(BANNER);
  54. }
  55. static void sbi_boot_print_general(struct sbi_scratch *scratch)
  56. {
  57. char str[128];
  58. const struct sbi_pmu_device *pdev;
  59. const struct sbi_hsm_device *hdev;
  60. const struct sbi_ipi_device *idev;
  61. const struct sbi_timer_device *tdev;
  62. const struct sbi_console_device *cdev;
  63. const struct sbi_system_reset_device *srdev;
  64. const struct sbi_system_suspend_device *susp_dev;
  65. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  66. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  67. return;
  68. /* Platform details */
  69. sbi_printf("Platform Name : %s\n",
  70. sbi_platform_name(plat));
  71. sbi_platform_get_features_str(plat, str, sizeof(str));
  72. sbi_printf("Platform Features : %s\n", str);
  73. sbi_printf("Platform HART Count : %u\n",
  74. sbi_platform_hart_count(plat));
  75. idev = sbi_ipi_get_device();
  76. sbi_printf("Platform IPI Device : %s\n",
  77. (idev) ? idev->name : "---");
  78. tdev = sbi_timer_get_device();
  79. sbi_printf("Platform Timer Device : %s @ %luHz\n",
  80. (tdev) ? tdev->name : "---",
  81. (tdev) ? tdev->timer_freq : 0);
  82. cdev = sbi_console_get_device();
  83. sbi_printf("Platform Console Device : %s\n",
  84. (cdev) ? cdev->name : "---");
  85. hdev = sbi_hsm_get_device();
  86. sbi_printf("Platform HSM Device : %s\n",
  87. (hdev) ? hdev->name : "---");
  88. pdev = sbi_pmu_get_device();
  89. sbi_printf("Platform PMU Device : %s\n",
  90. (pdev) ? pdev->name : "---");
  91. srdev = sbi_system_reset_get_device(SBI_SRST_RESET_TYPE_COLD_REBOOT, 0);
  92. sbi_printf("Platform Reboot Device : %s\n",
  93. (srdev) ? srdev->name : "---");
  94. srdev = sbi_system_reset_get_device(SBI_SRST_RESET_TYPE_SHUTDOWN, 0);
  95. sbi_printf("Platform Shutdown Device : %s\n",
  96. (srdev) ? srdev->name : "---");
  97. susp_dev = sbi_system_suspend_get_device();
  98. sbi_printf("Platform Suspend Device : %s\n",
  99. (susp_dev) ? susp_dev->name : "---");
  100. /* Firmware details */
  101. sbi_printf("Firmware Base : 0x%lx\n", scratch->fw_start);
  102. sbi_printf("Firmware Size : %d KB\n",
  103. (u32)(scratch->fw_size / 1024));
  104. sbi_printf("Firmware RW Offset : 0x%lx\n", scratch->fw_rw_offset);
  105. /* SBI details */
  106. sbi_printf("Runtime SBI Version : %d.%d\n",
  107. sbi_ecall_version_major(), sbi_ecall_version_minor());
  108. sbi_printf("\n");
  109. }
  110. static void sbi_boot_print_domains(struct sbi_scratch *scratch)
  111. {
  112. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  113. return;
  114. /* Domain details */
  115. sbi_domain_dump_all(" ");
  116. }
  117. static void sbi_boot_print_hart(struct sbi_scratch *scratch, u32 hartid)
  118. {
  119. int xlen;
  120. char str[128];
  121. const struct sbi_domain *dom = sbi_domain_thishart_ptr();
  122. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  123. return;
  124. /* Determine MISA XLEN and MISA string */
  125. xlen = misa_xlen();
  126. if (xlen < 1) {
  127. sbi_printf("Error %d getting MISA XLEN\n", xlen);
  128. sbi_hart_hang();
  129. }
  130. /* Boot HART details */
  131. sbi_printf("Boot HART ID : %u\n", hartid);
  132. sbi_printf("Boot HART Domain : %s\n", dom->name);
  133. sbi_hart_get_priv_version_str(scratch, str, sizeof(str));
  134. sbi_printf("Boot HART Priv Version : %s\n", str);
  135. misa_string(xlen, str, sizeof(str));
  136. sbi_printf("Boot HART Base ISA : %s\n", str);
  137. sbi_hart_get_extensions_str(scratch, str, sizeof(str));
  138. sbi_printf("Boot HART ISA Extensions : %s\n", str);
  139. sbi_printf("Boot HART PMP Count : %d\n",
  140. sbi_hart_pmp_count(scratch));
  141. sbi_printf("Boot HART PMP Granularity : %lu\n",
  142. sbi_hart_pmp_granularity(scratch));
  143. sbi_printf("Boot HART PMP Address Bits: %d\n",
  144. sbi_hart_pmp_addrbits(scratch));
  145. sbi_printf("Boot HART MHPM Count : %d\n",
  146. sbi_hart_mhpm_count(scratch));
  147. sbi_hart_delegation_dump(scratch, "Boot HART ", " ");
  148. }
  149. static spinlock_t coldboot_lock = SPIN_LOCK_INITIALIZER;
  150. static struct sbi_hartmask coldboot_wait_hmask = { 0 };
  151. static unsigned long coldboot_done;
  152. static void wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
  153. {
  154. unsigned long saved_mie, cmip;
  155. /* Save MIE CSR */
  156. saved_mie = csr_read(CSR_MIE);
  157. /* Set MSIE and MEIE bits to receive IPI */
  158. csr_set(CSR_MIE, MIP_MSIP | MIP_MEIP);
  159. /* Acquire coldboot lock */
  160. spin_lock(&coldboot_lock);
  161. /* Mark current HART as waiting */
  162. sbi_hartmask_set_hart(hartid, &coldboot_wait_hmask);
  163. /* Release coldboot lock */
  164. spin_unlock(&coldboot_lock);
  165. /* Wait for coldboot to finish using WFI */
  166. while (!__smp_load_acquire(&coldboot_done)) {
  167. do {
  168. wfi();
  169. cmip = csr_read(CSR_MIP);
  170. } while (!(cmip & (MIP_MSIP | MIP_MEIP)));
  171. };
  172. /* Acquire coldboot lock */
  173. spin_lock(&coldboot_lock);
  174. /* Unmark current HART as waiting */
  175. sbi_hartmask_clear_hart(hartid, &coldboot_wait_hmask);
  176. /* Release coldboot lock */
  177. spin_unlock(&coldboot_lock);
  178. /* Restore MIE CSR */
  179. csr_write(CSR_MIE, saved_mie);
  180. /*
  181. * The wait for coldboot is common for both warm startup and
  182. * warm resume path so clearing IPI here would result in losing
  183. * an IPI in warm resume path.
  184. *
  185. * Also, the sbi_platform_ipi_init() called from sbi_ipi_init()
  186. * will automatically clear IPI for current HART.
  187. */
  188. }
  189. static void wake_coldboot_harts(struct sbi_scratch *scratch, u32 hartid)
  190. {
  191. /* Mark coldboot done */
  192. __smp_store_release(&coldboot_done, 1);
  193. /* Acquire coldboot lock */
  194. spin_lock(&coldboot_lock);
  195. /* Send an IPI to all HARTs waiting for coldboot */
  196. for (u32 i = 0; i <= sbi_scratch_last_hartid(); i++) {
  197. if ((i != hartid) &&
  198. sbi_hartmask_test_hart(i, &coldboot_wait_hmask))
  199. sbi_ipi_raw_send(i);
  200. }
  201. /* Release coldboot lock */
  202. spin_unlock(&coldboot_lock);
  203. }
  204. static unsigned long init_count_offset;
  205. static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
  206. {
  207. int rc;
  208. unsigned long *init_count;
  209. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  210. /* Note: This has to be first thing in coldboot init sequence */
  211. rc = sbi_scratch_init(scratch);
  212. if (rc)
  213. sbi_hart_hang();
  214. /* Note: This has to be second thing in coldboot init sequence */
  215. rc = sbi_domain_init(scratch, hartid);
  216. if (rc)
  217. sbi_hart_hang();
  218. init_count_offset = sbi_scratch_alloc_offset(__SIZEOF_POINTER__);
  219. if (!init_count_offset)
  220. sbi_hart_hang();
  221. rc = sbi_hsm_init(scratch, hartid, true);
  222. if (rc)
  223. sbi_hart_hang();
  224. rc = sbi_platform_early_init(plat, true);
  225. if (rc)
  226. sbi_hart_hang();
  227. rc = sbi_hart_init(scratch, true);
  228. if (rc)
  229. sbi_hart_hang();
  230. rc = sbi_console_init(scratch);
  231. if (rc)
  232. sbi_hart_hang();
  233. rc = sbi_pmu_init(scratch, true);
  234. if (rc)
  235. sbi_hart_hang();
  236. sbi_boot_print_banner(scratch);
  237. rc = sbi_irqchip_init(scratch, true);
  238. if (rc) {
  239. sbi_printf("%s: irqchip init failed (error %d)\n",
  240. __func__, rc);
  241. sbi_hart_hang();
  242. }
  243. rc = sbi_ipi_init(scratch, true);
  244. if (rc) {
  245. sbi_printf("%s: ipi init failed (error %d)\n", __func__, rc);
  246. sbi_hart_hang();
  247. }
  248. rc = sbi_tlb_init(scratch, true);
  249. if (rc) {
  250. sbi_printf("%s: tlb init failed (error %d)\n", __func__, rc);
  251. sbi_hart_hang();
  252. }
  253. rc = sbi_timer_init(scratch, true);
  254. if (rc) {
  255. sbi_printf("%s: timer init failed (error %d)\n", __func__, rc);
  256. sbi_hart_hang();
  257. }
  258. rc = sbi_ecall_init();
  259. if (rc) {
  260. sbi_printf("%s: ecall init failed (error %d)\n", __func__, rc);
  261. sbi_hart_hang();
  262. }
  263. /*
  264. * Note: Finalize domains after HSM initialization so that we
  265. * can startup non-root domains.
  266. * Note: Finalize domains before HART PMP configuration so
  267. * that we use correct domain for configuring PMP.
  268. */
  269. rc = sbi_domain_finalize(scratch, hartid);
  270. if (rc) {
  271. sbi_printf("%s: domain finalize failed (error %d)\n",
  272. __func__, rc);
  273. sbi_hart_hang();
  274. }
  275. rc = sbi_hart_pmp_configure(scratch);
  276. if (rc) {
  277. sbi_printf("%s: PMP configure failed (error %d)\n",
  278. __func__, rc);
  279. sbi_hart_hang();
  280. }
  281. /*
  282. * Note: Platform final initialization should be last so that
  283. * it sees correct domain assignment and PMP configuration.
  284. */
  285. rc = sbi_platform_final_init(plat, true);
  286. if (rc) {
  287. sbi_printf("%s: platform final init failed (error %d)\n",
  288. __func__, rc);
  289. sbi_hart_hang();
  290. }
  291. sbi_boot_print_general(scratch);
  292. sbi_boot_print_domains(scratch);
  293. sbi_boot_print_hart(scratch, hartid);
  294. wake_coldboot_harts(scratch, hartid);
  295. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  296. (*init_count)++;
  297. sbi_hsm_hart_start_finish(scratch, hartid);
  298. }
  299. static void __noreturn init_warm_startup(struct sbi_scratch *scratch,
  300. u32 hartid)
  301. {
  302. int rc;
  303. unsigned long *init_count;
  304. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  305. if (!init_count_offset)
  306. sbi_hart_hang();
  307. rc = sbi_hsm_init(scratch, hartid, false);
  308. if (rc)
  309. sbi_hart_hang();
  310. rc = sbi_platform_early_init(plat, false);
  311. if (rc)
  312. sbi_hart_hang();
  313. rc = sbi_hart_init(scratch, false);
  314. if (rc)
  315. sbi_hart_hang();
  316. rc = sbi_pmu_init(scratch, false);
  317. if (rc)
  318. sbi_hart_hang();
  319. rc = sbi_irqchip_init(scratch, false);
  320. if (rc)
  321. sbi_hart_hang();
  322. rc = sbi_ipi_init(scratch, false);
  323. if (rc)
  324. sbi_hart_hang();
  325. rc = sbi_tlb_init(scratch, false);
  326. if (rc)
  327. sbi_hart_hang();
  328. rc = sbi_timer_init(scratch, false);
  329. if (rc)
  330. sbi_hart_hang();
  331. rc = sbi_hart_pmp_configure(scratch);
  332. if (rc)
  333. sbi_hart_hang();
  334. rc = sbi_platform_final_init(plat, false);
  335. if (rc)
  336. sbi_hart_hang();
  337. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  338. (*init_count)++;
  339. sbi_hsm_hart_start_finish(scratch, hartid);
  340. }
  341. static void __noreturn init_warm_resume(struct sbi_scratch *scratch,
  342. u32 hartid)
  343. {
  344. int rc;
  345. sbi_hsm_hart_resume_start(scratch);
  346. rc = sbi_hart_reinit(scratch);
  347. if (rc)
  348. sbi_hart_hang();
  349. rc = sbi_hart_pmp_configure(scratch);
  350. if (rc)
  351. sbi_hart_hang();
  352. sbi_hsm_hart_resume_finish(scratch, hartid);
  353. }
  354. static void __noreturn init_warmboot(struct sbi_scratch *scratch, u32 hartid)
  355. {
  356. int hstate;
  357. wait_for_coldboot(scratch, hartid);
  358. hstate = sbi_hsm_hart_get_state(sbi_domain_thishart_ptr(), hartid);
  359. if (hstate < 0)
  360. sbi_hart_hang();
  361. if (hstate == SBI_HSM_STATE_SUSPENDED) {
  362. init_warm_resume(scratch, hartid);
  363. } else {
  364. sbi_ipi_raw_clear(hartid);
  365. init_warm_startup(scratch, hartid);
  366. }
  367. }
  368. static atomic_t coldboot_lottery = ATOMIC_INITIALIZER(0);
  369. /**
  370. * Initialize OpenSBI library for current HART and jump to next
  371. * booting stage.
  372. *
  373. * The function expects following:
  374. * 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
  375. * 2. Stack pointer (SP) is setup for current HART
  376. * 3. Interrupts are disabled in MSTATUS CSR
  377. * 4. All interrupts are disabled in MIE CSR
  378. *
  379. * @param scratch pointer to sbi_scratch of current HART
  380. */
  381. void __noreturn sbi_init(struct sbi_scratch *scratch)
  382. {
  383. bool next_mode_supported = false;
  384. bool coldboot = false;
  385. u32 hartid = current_hartid();
  386. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  387. if ((SBI_HARTMASK_MAX_BITS <= hartid) ||
  388. sbi_platform_hart_invalid(plat, hartid))
  389. sbi_hart_hang();
  390. switch (scratch->next_mode) {
  391. case PRV_M:
  392. next_mode_supported = true;
  393. break;
  394. case PRV_S:
  395. if (misa_extension('S'))
  396. next_mode_supported = true;
  397. break;
  398. case PRV_U:
  399. if (misa_extension('U'))
  400. next_mode_supported = true;
  401. break;
  402. default:
  403. sbi_hart_hang();
  404. }
  405. /*
  406. * Only the HART supporting privilege mode specified in the
  407. * scratch->next_mode should be allowed to become the coldboot
  408. * HART because the coldboot HART will be directly jumping to
  409. * the next booting stage.
  410. *
  411. * We use a lottery mechanism to select coldboot HART among
  412. * HARTs which satisfy above condition.
  413. */
  414. if (sbi_platform_cold_boot_allowed(plat, hartid)) {
  415. if (next_mode_supported &&
  416. atomic_xchg(&coldboot_lottery, 1) == 0)
  417. coldboot = true;
  418. }
  419. /*
  420. * Do platform specific nascent (very early) initialization so
  421. * that platform can initialize platform specific per-HART CSRs
  422. * or per-HART devices.
  423. */
  424. if (sbi_platform_nascent_init(plat))
  425. sbi_hart_hang();
  426. if (coldboot)
  427. init_coldboot(scratch, hartid);
  428. else
  429. init_warmboot(scratch, hartid);
  430. }
  431. unsigned long sbi_init_count(u32 hartid)
  432. {
  433. struct sbi_scratch *scratch;
  434. unsigned long *init_count;
  435. if (!init_count_offset)
  436. return 0;
  437. scratch = sbi_hartid_to_scratch(hartid);
  438. if (!scratch)
  439. return 0;
  440. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  441. return *init_count;
  442. }
  443. /**
  444. * Exit OpenSBI library for current HART and stop HART
  445. *
  446. * The function expects following:
  447. * 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
  448. * 2. Stack pointer (SP) is setup for current HART
  449. *
  450. * @param scratch pointer to sbi_scratch of current HART
  451. */
  452. void __noreturn sbi_exit(struct sbi_scratch *scratch)
  453. {
  454. u32 hartid = current_hartid();
  455. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  456. if (sbi_platform_hart_invalid(plat, hartid))
  457. sbi_hart_hang();
  458. sbi_platform_early_exit(plat);
  459. sbi_pmu_exit(scratch);
  460. sbi_timer_exit(scratch);
  461. sbi_ipi_exit(scratch);
  462. sbi_irqchip_exit(scratch);
  463. sbi_platform_final_exit(plat);
  464. sbi_hsm_exit(scratch);
  465. }