sbi_init.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565
  1. /*
  2. * SPDX-License-Identifier: BSD-2-Clause
  3. *
  4. * Copyright (c) 2019 Western Digital Corporation or its affiliates.
  5. *
  6. * Authors:
  7. * Anup Patel <anup.patel@wdc.com>
  8. */
  9. #include <sbi/riscv_asm.h>
  10. #include <sbi/riscv_atomic.h>
  11. #include <sbi/riscv_barrier.h>
  12. #include <sbi/riscv_locks.h>
  13. #include <sbi/sbi_console.h>
  14. #include <sbi/sbi_domain.h>
  15. #include <sbi/sbi_ecall.h>
  16. #include <sbi/sbi_hart.h>
  17. #include <sbi/sbi_hartmask.h>
  18. #include <sbi/sbi_hsm.h>
  19. #include <sbi/sbi_ipi.h>
  20. #include <sbi/sbi_irqchip.h>
  21. #include <sbi/sbi_platform.h>
  22. #include <sbi/sbi_pmu.h>
  23. #include <sbi/sbi_system.h>
  24. #include <sbi/sbi_string.h>
  25. #include <sbi/sbi_timer.h>
  26. #include <sbi/sbi_tlb.h>
  27. #include <sbi/sbi_version.h>
  28. #define BANNER \
  29. " ____ _____ ____ _____\n" \
  30. " / __ \\ / ____| _ \\_ _|\n" \
  31. " | | | |_ __ ___ _ __ | (___ | |_) || |\n" \
  32. " | | | | '_ \\ / _ \\ '_ \\ \\___ \\| _ < | |\n" \
  33. " | |__| | |_) | __/ | | |____) | |_) || |_\n" \
  34. " \\____/| .__/ \\___|_| |_|_____/|____/_____|\n" \
  35. " | |\n" \
  36. " |_|\n\n"
  37. static void sbi_boot_print_banner(struct sbi_scratch *scratch)
  38. {
  39. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  40. return;
  41. #ifdef OPENSBI_VERSION_GIT
  42. sbi_printf("\nOpenSBI %s\n", OPENSBI_VERSION_GIT);
  43. #else
  44. sbi_printf("\nOpenSBI v%d.%d\n", OPENSBI_VERSION_MAJOR,
  45. OPENSBI_VERSION_MINOR);
  46. #endif
  47. #ifdef OPENSBI_BUILD_TIME_STAMP
  48. sbi_printf("Build time: %s\n", OPENSBI_BUILD_TIME_STAMP);
  49. #endif
  50. #ifdef OPENSBI_BUILD_COMPILER_VERSION
  51. sbi_printf("Build compiler: %s\n", OPENSBI_BUILD_COMPILER_VERSION);
  52. #endif
  53. sbi_printf(BANNER);
  54. }
  55. static void sbi_boot_print_general(struct sbi_scratch *scratch)
  56. {
  57. char str[128];
  58. const struct sbi_pmu_device *pdev;
  59. const struct sbi_hsm_device *hdev;
  60. const struct sbi_ipi_device *idev;
  61. const struct sbi_timer_device *tdev;
  62. const struct sbi_console_device *cdev;
  63. const struct sbi_system_reset_device *srdev;
  64. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  65. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  66. return;
  67. /* Platform details */
  68. sbi_printf("Platform Name : %s\n",
  69. sbi_platform_name(plat));
  70. sbi_platform_get_features_str(plat, str, sizeof(str));
  71. sbi_printf("Platform Features : %s\n", str);
  72. sbi_printf("Platform HART Count : %u\n",
  73. sbi_platform_hart_count(plat));
  74. idev = sbi_ipi_get_device();
  75. sbi_printf("Platform IPI Device : %s\n",
  76. (idev) ? idev->name : "---");
  77. tdev = sbi_timer_get_device();
  78. sbi_printf("Platform Timer Device : %s @ %luHz\n",
  79. (tdev) ? tdev->name : "---",
  80. (tdev) ? tdev->timer_freq : 0);
  81. cdev = sbi_console_get_device();
  82. sbi_printf("Platform Console Device : %s\n",
  83. (cdev) ? cdev->name : "---");
  84. hdev = sbi_hsm_get_device();
  85. sbi_printf("Platform HSM Device : %s\n",
  86. (hdev) ? hdev->name : "---");
  87. pdev = sbi_pmu_get_device();
  88. sbi_printf("Platform PMU Device : %s\n",
  89. (pdev) ? pdev->name : "---");
  90. srdev = sbi_system_reset_get_device(SBI_SRST_RESET_TYPE_COLD_REBOOT, 0);
  91. sbi_printf("Platform Reboot Device : %s\n",
  92. (srdev) ? srdev->name : "---");
  93. srdev = sbi_system_reset_get_device(SBI_SRST_RESET_TYPE_SHUTDOWN, 0);
  94. sbi_printf("Platform Shutdown Device : %s\n",
  95. (srdev) ? srdev->name : "---");
  96. /* Firmware details */
  97. sbi_printf("Firmware Base : 0x%lx\n", scratch->fw_start);
  98. sbi_printf("Firmware Size : %d KB\n",
  99. (u32)(scratch->fw_size / 1024));
  100. /* SBI details */
  101. sbi_printf("Runtime SBI Version : %d.%d\n",
  102. sbi_ecall_version_major(), sbi_ecall_version_minor());
  103. sbi_printf("\n");
  104. }
  105. static void sbi_boot_print_domains(struct sbi_scratch *scratch)
  106. {
  107. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  108. return;
  109. /* Domain details */
  110. sbi_domain_dump_all(" ");
  111. }
  112. static void sbi_boot_print_hart(struct sbi_scratch *scratch, u32 hartid)
  113. {
  114. int xlen;
  115. char str[128];
  116. const struct sbi_domain *dom = sbi_domain_thishart_ptr();
  117. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  118. return;
  119. /* Determine MISA XLEN and MISA string */
  120. xlen = misa_xlen();
  121. if (xlen < 1) {
  122. sbi_printf("Error %d getting MISA XLEN\n", xlen);
  123. sbi_hart_hang();
  124. }
  125. /* Boot HART details */
  126. sbi_printf("Boot HART ID : %u\n", hartid);
  127. sbi_printf("Boot HART Domain : %s\n", dom->name);
  128. sbi_hart_get_priv_version_str(scratch, str, sizeof(str));
  129. sbi_printf("Boot HART Priv Version : %s\n", str);
  130. misa_string(xlen, str, sizeof(str));
  131. sbi_printf("Boot HART Base ISA : %s\n", str);
  132. sbi_hart_get_extensions_str(scratch, str, sizeof(str));
  133. sbi_printf("Boot HART ISA Extensions : %s\n", str);
  134. sbi_printf("Boot HART PMP Count : %d\n",
  135. sbi_hart_pmp_count(scratch));
  136. sbi_printf("Boot HART PMP Granularity : %lu\n",
  137. sbi_hart_pmp_granularity(scratch));
  138. sbi_printf("Boot HART PMP Address Bits: %d\n",
  139. sbi_hart_pmp_addrbits(scratch));
  140. sbi_printf("Boot HART MHPM Count : %d\n",
  141. sbi_hart_mhpm_count(scratch));
  142. sbi_hart_delegation_dump(scratch, "Boot HART ", " ");
  143. }
  144. static spinlock_t coldboot_lock = SPIN_LOCK_INITIALIZER;
  145. static struct sbi_hartmask coldboot_wait_hmask = { 0 };
  146. static unsigned long coldboot_done;
  147. static void wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
  148. {
  149. unsigned long saved_mie, cmip;
  150. /* Save MIE CSR */
  151. saved_mie = csr_read(CSR_MIE);
  152. /* Set MSIE and MEIE bits to receive IPI */
  153. csr_set(CSR_MIE, MIP_MSIP | MIP_MEIP);
  154. /* Acquire coldboot lock */
  155. spin_lock(&coldboot_lock);
  156. /* Mark current HART as waiting */
  157. sbi_hartmask_set_hart(hartid, &coldboot_wait_hmask);
  158. /* Release coldboot lock */
  159. spin_unlock(&coldboot_lock);
  160. /* Wait for coldboot to finish using WFI */
  161. while (!__smp_load_acquire(&coldboot_done)) {
  162. do {
  163. wfi();
  164. cmip = csr_read(CSR_MIP);
  165. } while (!(cmip & (MIP_MSIP | MIP_MEIP)));
  166. };
  167. /* Acquire coldboot lock */
  168. spin_lock(&coldboot_lock);
  169. /* Unmark current HART as waiting */
  170. sbi_hartmask_clear_hart(hartid, &coldboot_wait_hmask);
  171. /* Release coldboot lock */
  172. spin_unlock(&coldboot_lock);
  173. /* Restore MIE CSR */
  174. csr_write(CSR_MIE, saved_mie);
  175. /*
  176. * The wait for coldboot is common for both warm startup and
  177. * warm resume path so clearing IPI here would result in losing
  178. * an IPI in warm resume path.
  179. *
  180. * Also, the sbi_platform_ipi_init() called from sbi_ipi_init()
  181. * will automatically clear IPI for current HART.
  182. */
  183. }
  184. static void wake_coldboot_harts(struct sbi_scratch *scratch, u32 hartid)
  185. {
  186. /* Mark coldboot done */
  187. __smp_store_release(&coldboot_done, 1);
  188. /* Acquire coldboot lock */
  189. spin_lock(&coldboot_lock);
  190. /* Send an IPI to all HARTs waiting for coldboot */
  191. for (u32 i = 0; i <= sbi_scratch_last_hartid(); i++) {
  192. if ((i != hartid) &&
  193. sbi_hartmask_test_hart(i, &coldboot_wait_hmask))
  194. sbi_ipi_raw_send(i);
  195. }
  196. /* Release coldboot lock */
  197. spin_unlock(&coldboot_lock);
  198. }
  199. static unsigned long init_count_offset;
  200. static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
  201. {
  202. int rc;
  203. unsigned long *init_count;
  204. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  205. /* Note: This has to be first thing in coldboot init sequence */
  206. rc = sbi_scratch_init(scratch);
  207. if (rc)
  208. sbi_hart_hang();
  209. /* Note: This has to be second thing in coldboot init sequence */
  210. rc = sbi_domain_init(scratch, hartid);
  211. if (rc)
  212. sbi_hart_hang();
  213. init_count_offset = sbi_scratch_alloc_offset(__SIZEOF_POINTER__);
  214. if (!init_count_offset)
  215. sbi_hart_hang();
  216. rc = sbi_hsm_init(scratch, hartid, TRUE);
  217. if (rc)
  218. sbi_hart_hang();
  219. rc = sbi_platform_early_init(plat, TRUE);
  220. if (rc)
  221. sbi_hart_hang();
  222. rc = sbi_hart_init(scratch, TRUE);
  223. if (rc)
  224. sbi_hart_hang();
  225. rc = sbi_console_init(scratch);
  226. if (rc)
  227. sbi_hart_hang();
  228. rc = sbi_pmu_init(scratch, TRUE);
  229. if (rc)
  230. sbi_hart_hang();
  231. sbi_boot_print_banner(scratch);
  232. rc = sbi_irqchip_init(scratch, TRUE);
  233. if (rc) {
  234. sbi_printf("%s: irqchip init failed (error %d)\n",
  235. __func__, rc);
  236. sbi_hart_hang();
  237. }
  238. rc = sbi_ipi_init(scratch, TRUE);
  239. if (rc) {
  240. sbi_printf("%s: ipi init failed (error %d)\n", __func__, rc);
  241. sbi_hart_hang();
  242. }
  243. rc = sbi_tlb_init(scratch, TRUE);
  244. if (rc) {
  245. sbi_printf("%s: tlb init failed (error %d)\n", __func__, rc);
  246. sbi_hart_hang();
  247. }
  248. rc = sbi_timer_init(scratch, TRUE);
  249. if (rc) {
  250. sbi_printf("%s: timer init failed (error %d)\n", __func__, rc);
  251. sbi_hart_hang();
  252. }
  253. rc = sbi_ecall_init();
  254. if (rc) {
  255. sbi_printf("%s: ecall init failed (error %d)\n", __func__, rc);
  256. sbi_hart_hang();
  257. }
  258. /*
  259. * Note: Finalize domains after HSM initialization so that we
  260. * can startup non-root domains.
  261. * Note: Finalize domains before HART PMP configuration so
  262. * that we use correct domain for configuring PMP.
  263. */
  264. rc = sbi_domain_finalize(scratch, hartid);
  265. if (rc) {
  266. sbi_printf("%s: domain finalize failed (error %d)\n",
  267. __func__, rc);
  268. sbi_hart_hang();
  269. }
  270. rc = sbi_hart_pmp_configure(scratch);
  271. if (rc) {
  272. sbi_printf("%s: PMP configure failed (error %d)\n",
  273. __func__, rc);
  274. sbi_hart_hang();
  275. }
  276. /*
  277. * Note: Platform final initialization should be last so that
  278. * it sees correct domain assignment and PMP configuration.
  279. */
  280. rc = sbi_platform_final_init(plat, TRUE);
  281. if (rc) {
  282. sbi_printf("%s: platform final init failed (error %d)\n",
  283. __func__, rc);
  284. sbi_hart_hang();
  285. }
  286. sbi_boot_print_general(scratch);
  287. sbi_boot_print_domains(scratch);
  288. sbi_boot_print_hart(scratch, hartid);
  289. wake_coldboot_harts(scratch, hartid);
  290. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  291. (*init_count)++;
  292. sbi_hsm_prepare_next_jump(scratch, hartid);
  293. sbi_hart_switch_mode(hartid, scratch->next_arg1, scratch->next_addr,
  294. scratch->next_mode, FALSE);
  295. }
  296. static void init_warm_startup(struct sbi_scratch *scratch, u32 hartid)
  297. {
  298. int rc;
  299. unsigned long *init_count;
  300. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  301. if (!init_count_offset)
  302. sbi_hart_hang();
  303. rc = sbi_hsm_init(scratch, hartid, FALSE);
  304. if (rc)
  305. sbi_hart_hang();
  306. rc = sbi_platform_early_init(plat, FALSE);
  307. if (rc)
  308. sbi_hart_hang();
  309. rc = sbi_hart_init(scratch, FALSE);
  310. if (rc)
  311. sbi_hart_hang();
  312. rc = sbi_pmu_init(scratch, FALSE);
  313. if (rc)
  314. sbi_hart_hang();
  315. rc = sbi_irqchip_init(scratch, FALSE);
  316. if (rc)
  317. sbi_hart_hang();
  318. rc = sbi_ipi_init(scratch, FALSE);
  319. if (rc)
  320. sbi_hart_hang();
  321. rc = sbi_tlb_init(scratch, FALSE);
  322. if (rc)
  323. sbi_hart_hang();
  324. rc = sbi_timer_init(scratch, FALSE);
  325. if (rc)
  326. sbi_hart_hang();
  327. rc = sbi_hart_pmp_configure(scratch);
  328. if (rc)
  329. sbi_hart_hang();
  330. rc = sbi_platform_final_init(plat, FALSE);
  331. if (rc)
  332. sbi_hart_hang();
  333. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  334. (*init_count)++;
  335. sbi_hsm_prepare_next_jump(scratch, hartid);
  336. }
  337. static void init_warm_resume(struct sbi_scratch *scratch)
  338. {
  339. int rc;
  340. sbi_hsm_hart_resume_start(scratch);
  341. rc = sbi_hart_reinit(scratch);
  342. if (rc)
  343. sbi_hart_hang();
  344. rc = sbi_hart_pmp_configure(scratch);
  345. if (rc)
  346. sbi_hart_hang();
  347. sbi_hsm_hart_resume_finish(scratch);
  348. }
  349. static void __noreturn init_warmboot(struct sbi_scratch *scratch, u32 hartid)
  350. {
  351. int hstate;
  352. wait_for_coldboot(scratch, hartid);
  353. hstate = sbi_hsm_hart_get_state(sbi_domain_thishart_ptr(), hartid);
  354. if (hstate < 0)
  355. sbi_hart_hang();
  356. if (hstate == SBI_HSM_STATE_SUSPENDED)
  357. init_warm_resume(scratch);
  358. else
  359. init_warm_startup(scratch, hartid);
  360. sbi_hart_switch_mode(hartid, scratch->next_arg1,
  361. scratch->next_addr,
  362. scratch->next_mode, FALSE);
  363. }
  364. static atomic_t coldboot_lottery = ATOMIC_INITIALIZER(0);
  365. /**
  366. * Initialize OpenSBI library for current HART and jump to next
  367. * booting stage.
  368. *
  369. * The function expects following:
  370. * 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
  371. * 2. Stack pointer (SP) is setup for current HART
  372. * 3. Interrupts are disabled in MSTATUS CSR
  373. * 4. All interrupts are disabled in MIE CSR
  374. *
  375. * @param scratch pointer to sbi_scratch of current HART
  376. */
  377. void __noreturn sbi_init(struct sbi_scratch *scratch)
  378. {
  379. bool next_mode_supported = FALSE;
  380. bool coldboot = FALSE;
  381. u32 hartid = current_hartid();
  382. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  383. if ((SBI_HARTMASK_MAX_BITS <= hartid) ||
  384. sbi_platform_hart_invalid(plat, hartid))
  385. sbi_hart_hang();
  386. switch (scratch->next_mode) {
  387. case PRV_M:
  388. next_mode_supported = TRUE;
  389. break;
  390. case PRV_S:
  391. if (misa_extension('S'))
  392. next_mode_supported = TRUE;
  393. break;
  394. case PRV_U:
  395. if (misa_extension('U'))
  396. next_mode_supported = TRUE;
  397. break;
  398. default:
  399. sbi_hart_hang();
  400. }
  401. /*
  402. * Only the HART supporting privilege mode specified in the
  403. * scratch->next_mode should be allowed to become the coldboot
  404. * HART because the coldboot HART will be directly jumping to
  405. * the next booting stage.
  406. *
  407. * We use a lottery mechanism to select coldboot HART among
  408. * HARTs which satisfy above condition.
  409. */
  410. if (next_mode_supported && atomic_xchg(&coldboot_lottery, 1) == 0)
  411. coldboot = TRUE;
  412. /*
  413. * Do platform specific nascent (very early) initialization so
  414. * that platform can initialize platform specific per-HART CSRs
  415. * or per-HART devices.
  416. */
  417. if (sbi_platform_nascent_init(plat))
  418. sbi_hart_hang();
  419. if (coldboot)
  420. init_coldboot(scratch, hartid);
  421. else
  422. init_warmboot(scratch, hartid);
  423. }
  424. unsigned long sbi_init_count(u32 hartid)
  425. {
  426. struct sbi_scratch *scratch;
  427. unsigned long *init_count;
  428. if (!init_count_offset)
  429. return 0;
  430. scratch = sbi_hartid_to_scratch(hartid);
  431. if (!scratch)
  432. return 0;
  433. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  434. return *init_count;
  435. }
  436. /**
  437. * Exit OpenSBI library for current HART and stop HART
  438. *
  439. * The function expects following:
  440. * 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
  441. * 2. Stack pointer (SP) is setup for current HART
  442. *
  443. * @param scratch pointer to sbi_scratch of current HART
  444. */
  445. void __noreturn sbi_exit(struct sbi_scratch *scratch)
  446. {
  447. u32 hartid = current_hartid();
  448. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  449. if (sbi_platform_hart_invalid(plat, hartid))
  450. sbi_hart_hang();
  451. sbi_platform_early_exit(plat);
  452. sbi_pmu_exit(scratch);
  453. sbi_timer_exit(scratch);
  454. sbi_ipi_exit(scratch);
  455. sbi_irqchip_exit(scratch);
  456. sbi_platform_final_exit(plat);
  457. sbi_hsm_exit(scratch);
  458. }