sbi_init.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538
  1. /*
  2. * SPDX-License-Identifier: BSD-2-Clause
  3. *
  4. * Copyright (c) 2019 Western Digital Corporation or its affiliates.
  5. *
  6. * Authors:
  7. * Anup Patel <anup.patel@wdc.com>
  8. */
  9. #include <sbi/riscv_asm.h>
  10. #include <sbi/riscv_atomic.h>
  11. #include <sbi/riscv_barrier.h>
  12. #include <sbi/riscv_locks.h>
  13. #include <sbi/sbi_console.h>
  14. #include <sbi/sbi_domain.h>
  15. #include <sbi/sbi_ecall.h>
  16. #include <sbi/sbi_hart.h>
  17. #include <sbi/sbi_hartmask.h>
  18. #include <sbi/sbi_hsm.h>
  19. #include <sbi/sbi_ipi.h>
  20. #include <sbi/sbi_platform.h>
  21. #include <sbi/sbi_pmu.h>
  22. #include <sbi/sbi_system.h>
  23. #include <sbi/sbi_string.h>
  24. #include <sbi/sbi_timer.h>
  25. #include <sbi/sbi_tlb.h>
  26. #include <sbi/sbi_version.h>
  27. #define BANNER \
  28. " ____ _____ ____ _____\n" \
  29. " / __ \\ / ____| _ \\_ _|\n" \
  30. " | | | |_ __ ___ _ __ | (___ | |_) || |\n" \
  31. " | | | | '_ \\ / _ \\ '_ \\ \\___ \\| _ < | |\n" \
  32. " | |__| | |_) | __/ | | |____) | |_) || |_\n" \
  33. " \\____/| .__/ \\___|_| |_|_____/|____/_____|\n" \
  34. " | |\n" \
  35. " |_|\n\n"
  36. static void sbi_boot_print_banner(struct sbi_scratch *scratch)
  37. {
  38. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  39. return;
  40. #ifdef OPENSBI_VERSION_GIT
  41. sbi_printf("\nOpenSBI %s\n", OPENSBI_VERSION_GIT);
  42. #else
  43. sbi_printf("\nOpenSBI v%d.%d\n", OPENSBI_VERSION_MAJOR,
  44. OPENSBI_VERSION_MINOR);
  45. #endif
  46. sbi_printf(BANNER);
  47. }
  48. static void sbi_boot_print_general(struct sbi_scratch *scratch)
  49. {
  50. char str[128];
  51. const struct sbi_hsm_device *hdev;
  52. const struct sbi_ipi_device *idev;
  53. const struct sbi_timer_device *tdev;
  54. const struct sbi_console_device *cdev;
  55. const struct sbi_system_reset_device *srdev;
  56. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  57. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  58. return;
  59. /* Platform details */
  60. sbi_printf("Platform Name : %s\n",
  61. sbi_platform_name(plat));
  62. sbi_platform_get_features_str(plat, str, sizeof(str));
  63. sbi_printf("Platform Features : %s\n", str);
  64. sbi_printf("Platform HART Count : %u\n",
  65. sbi_platform_hart_count(plat));
  66. idev = sbi_ipi_get_device();
  67. sbi_printf("Platform IPI Device : %s\n",
  68. (idev) ? idev->name : "---");
  69. tdev = sbi_timer_get_device();
  70. sbi_printf("Platform Timer Device : %s\n",
  71. (tdev) ? tdev->name : "---");
  72. cdev = sbi_console_get_device();
  73. sbi_printf("Platform Console Device : %s\n",
  74. (cdev) ? cdev->name : "---");
  75. hdev = sbi_hsm_get_device();
  76. sbi_printf("Platform HSM Device : %s\n",
  77. (hdev) ? hdev->name : "---");
  78. srdev = sbi_system_reset_get_device();
  79. sbi_printf("Platform SysReset Device : %s\n",
  80. (srdev) ? srdev->name : "---");
  81. /* Firmware details */
  82. sbi_printf("Firmware Base : 0x%lx\n", scratch->fw_start);
  83. sbi_printf("Firmware Size : %d KB\n",
  84. (u32)(scratch->fw_size / 1024));
  85. /* SBI details */
  86. sbi_printf("Runtime SBI Version : %d.%d\n",
  87. sbi_ecall_version_major(), sbi_ecall_version_minor());
  88. sbi_printf("\n");
  89. }
  90. static void sbi_boot_print_domains(struct sbi_scratch *scratch)
  91. {
  92. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  93. return;
  94. /* Domain details */
  95. sbi_domain_dump_all(" ");
  96. }
  97. static void sbi_boot_print_hart(struct sbi_scratch *scratch, u32 hartid)
  98. {
  99. int xlen;
  100. char str[128];
  101. const struct sbi_domain *dom = sbi_domain_thishart_ptr();
  102. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  103. return;
  104. /* Determine MISA XLEN and MISA string */
  105. xlen = misa_xlen();
  106. if (xlen < 1) {
  107. sbi_printf("Error %d getting MISA XLEN\n", xlen);
  108. sbi_hart_hang();
  109. }
  110. /* Boot HART details */
  111. sbi_printf("Boot HART ID : %u\n", hartid);
  112. sbi_printf("Boot HART Domain : %s\n", dom->name);
  113. misa_string(xlen, str, sizeof(str));
  114. sbi_printf("Boot HART ISA : %s\n", str);
  115. sbi_hart_get_features_str(scratch, str, sizeof(str));
  116. sbi_printf("Boot HART Features : %s\n", str);
  117. sbi_printf("Boot HART PMP Count : %d\n",
  118. sbi_hart_pmp_count(scratch));
  119. sbi_printf("Boot HART PMP Granularity : %lu\n",
  120. sbi_hart_pmp_granularity(scratch));
  121. sbi_printf("Boot HART PMP Address Bits: %d\n",
  122. sbi_hart_pmp_addrbits(scratch));
  123. sbi_printf("Boot HART MHPM Count : %d\n",
  124. sbi_hart_mhpm_count(scratch));
  125. sbi_hart_delegation_dump(scratch, "Boot HART ", " ");
  126. }
  127. static spinlock_t coldboot_lock = SPIN_LOCK_INITIALIZER;
  128. static struct sbi_hartmask coldboot_wait_hmask = { 0 };
  129. static unsigned long coldboot_done;
  130. static void wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
  131. {
  132. unsigned long saved_mie, cmip;
  133. /* Save MIE CSR */
  134. saved_mie = csr_read(CSR_MIE);
  135. /* Set MSIE bit to receive IPI */
  136. csr_set(CSR_MIE, MIP_MSIP);
  137. /* Acquire coldboot lock */
  138. spin_lock(&coldboot_lock);
  139. /* Mark current HART as waiting */
  140. sbi_hartmask_set_hart(hartid, &coldboot_wait_hmask);
  141. /* Release coldboot lock */
  142. spin_unlock(&coldboot_lock);
  143. /* Wait for coldboot to finish using WFI */
  144. while (!__smp_load_acquire(&coldboot_done)) {
  145. do {
  146. wfi();
  147. cmip = csr_read(CSR_MIP);
  148. } while (!(cmip & MIP_MSIP));
  149. };
  150. /* Acquire coldboot lock */
  151. spin_lock(&coldboot_lock);
  152. /* Unmark current HART as waiting */
  153. sbi_hartmask_clear_hart(hartid, &coldboot_wait_hmask);
  154. /* Release coldboot lock */
  155. spin_unlock(&coldboot_lock);
  156. /* Restore MIE CSR */
  157. csr_write(CSR_MIE, saved_mie);
  158. /*
  159. * The wait for coldboot is common for both warm startup and
  160. * warm resume path so clearing IPI here would result in losing
  161. * an IPI in warm resume path.
  162. *
  163. * Also, the sbi_platform_ipi_init() called from sbi_ipi_init()
  164. * will automatically clear IPI for current HART.
  165. */
  166. }
  167. static void wake_coldboot_harts(struct sbi_scratch *scratch, u32 hartid)
  168. {
  169. /* Mark coldboot done */
  170. __smp_store_release(&coldboot_done, 1);
  171. /* Acquire coldboot lock */
  172. spin_lock(&coldboot_lock);
  173. /* Send an IPI to all HARTs waiting for coldboot */
  174. for (u32 i = 0; i <= sbi_scratch_last_hartid(); i++) {
  175. if ((i != hartid) &&
  176. sbi_hartmask_test_hart(i, &coldboot_wait_hmask))
  177. sbi_ipi_raw_send(i);
  178. }
  179. /* Release coldboot lock */
  180. spin_unlock(&coldboot_lock);
  181. }
  182. static unsigned long init_count_offset;
  183. static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
  184. {
  185. int rc;
  186. unsigned long *init_count;
  187. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  188. /* Note: This has to be first thing in coldboot init sequence */
  189. rc = sbi_scratch_init(scratch);
  190. if (rc)
  191. sbi_hart_hang();
  192. /* Note: This has to be second thing in coldboot init sequence */
  193. rc = sbi_domain_init(scratch, hartid);
  194. if (rc)
  195. sbi_hart_hang();
  196. init_count_offset = sbi_scratch_alloc_offset(__SIZEOF_POINTER__);
  197. if (!init_count_offset)
  198. sbi_hart_hang();
  199. rc = sbi_hsm_init(scratch, hartid, TRUE);
  200. if (rc)
  201. sbi_hart_hang();
  202. rc = sbi_platform_early_init(plat, TRUE);
  203. if (rc)
  204. sbi_hart_hang();
  205. rc = sbi_hart_init(scratch, TRUE);
  206. if (rc)
  207. sbi_hart_hang();
  208. rc = sbi_console_init(scratch);
  209. if (rc)
  210. sbi_hart_hang();
  211. rc = sbi_pmu_init(scratch, TRUE);
  212. if (rc)
  213. sbi_hart_hang();
  214. sbi_boot_print_banner(scratch);
  215. rc = sbi_platform_irqchip_init(plat, TRUE);
  216. if (rc) {
  217. sbi_printf("%s: platform irqchip init failed (error %d)\n",
  218. __func__, rc);
  219. sbi_hart_hang();
  220. }
  221. rc = sbi_ipi_init(scratch, TRUE);
  222. if (rc) {
  223. sbi_printf("%s: ipi init failed (error %d)\n", __func__, rc);
  224. sbi_hart_hang();
  225. }
  226. rc = sbi_tlb_init(scratch, TRUE);
  227. if (rc) {
  228. sbi_printf("%s: tlb init failed (error %d)\n", __func__, rc);
  229. sbi_hart_hang();
  230. }
  231. rc = sbi_timer_init(scratch, TRUE);
  232. if (rc) {
  233. sbi_printf("%s: timer init failed (error %d)\n", __func__, rc);
  234. sbi_hart_hang();
  235. }
  236. rc = sbi_ecall_init();
  237. if (rc) {
  238. sbi_printf("%s: ecall init failed (error %d)\n", __func__, rc);
  239. sbi_hart_hang();
  240. }
  241. sbi_boot_print_general(scratch);
  242. /*
  243. * Note: Finalize domains after HSM initialization so that we
  244. * can startup non-root domains.
  245. * Note: Finalize domains before HART PMP configuration so
  246. * that we use correct domain for configuring PMP.
  247. */
  248. rc = sbi_domain_finalize(scratch, hartid);
  249. if (rc) {
  250. sbi_printf("%s: domain finalize failed (error %d)\n",
  251. __func__, rc);
  252. sbi_hart_hang();
  253. }
  254. sbi_boot_print_domains(scratch);
  255. rc = sbi_hart_pmp_configure(scratch);
  256. if (rc) {
  257. sbi_printf("%s: PMP configure failed (error %d)\n",
  258. __func__, rc);
  259. sbi_hart_hang();
  260. }
  261. /*
  262. * Note: Platform final initialization should be last so that
  263. * it sees correct domain assignment and PMP configuration.
  264. */
  265. rc = sbi_platform_final_init(plat, TRUE);
  266. if (rc) {
  267. sbi_printf("%s: platform final init failed (error %d)\n",
  268. __func__, rc);
  269. sbi_hart_hang();
  270. }
  271. sbi_boot_print_hart(scratch, hartid);
  272. wake_coldboot_harts(scratch, hartid);
  273. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  274. (*init_count)++;
  275. sbi_hsm_prepare_next_jump(scratch, hartid);
  276. sbi_hart_switch_mode(hartid, scratch->next_arg1, scratch->next_addr,
  277. scratch->next_mode, FALSE);
  278. }
  279. static void init_warm_startup(struct sbi_scratch *scratch, u32 hartid)
  280. {
  281. int rc;
  282. unsigned long *init_count;
  283. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  284. if (!init_count_offset)
  285. sbi_hart_hang();
  286. rc = sbi_hsm_init(scratch, hartid, FALSE);
  287. if (rc)
  288. sbi_hart_hang();
  289. rc = sbi_platform_early_init(plat, FALSE);
  290. if (rc)
  291. sbi_hart_hang();
  292. rc = sbi_hart_init(scratch, FALSE);
  293. if (rc)
  294. sbi_hart_hang();
  295. rc = sbi_pmu_init(scratch, FALSE);
  296. if (rc)
  297. sbi_hart_hang();
  298. rc = sbi_platform_irqchip_init(plat, FALSE);
  299. if (rc)
  300. sbi_hart_hang();
  301. rc = sbi_ipi_init(scratch, FALSE);
  302. if (rc)
  303. sbi_hart_hang();
  304. rc = sbi_tlb_init(scratch, FALSE);
  305. if (rc)
  306. sbi_hart_hang();
  307. rc = sbi_timer_init(scratch, FALSE);
  308. if (rc)
  309. sbi_hart_hang();
  310. rc = sbi_hart_pmp_configure(scratch);
  311. if (rc)
  312. sbi_hart_hang();
  313. rc = sbi_platform_final_init(plat, FALSE);
  314. if (rc)
  315. sbi_hart_hang();
  316. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  317. (*init_count)++;
  318. sbi_hsm_prepare_next_jump(scratch, hartid);
  319. }
  320. static void init_warm_resume(struct sbi_scratch *scratch)
  321. {
  322. int rc;
  323. sbi_hsm_hart_resume_start(scratch);
  324. rc = sbi_hart_reinit(scratch);
  325. if (rc)
  326. sbi_hart_hang();
  327. rc = sbi_hart_pmp_configure(scratch);
  328. if (rc)
  329. sbi_hart_hang();
  330. sbi_hsm_hart_resume_finish(scratch);
  331. }
  332. static void __noreturn init_warmboot(struct sbi_scratch *scratch, u32 hartid)
  333. {
  334. int hstate;
  335. wait_for_coldboot(scratch, hartid);
  336. hstate = sbi_hsm_hart_get_state(sbi_domain_thishart_ptr(), hartid);
  337. if (hstate < 0)
  338. sbi_hart_hang();
  339. if (hstate == SBI_HSM_STATE_SUSPENDED)
  340. init_warm_resume(scratch);
  341. else
  342. init_warm_startup(scratch, hartid);
  343. sbi_hart_switch_mode(hartid, scratch->next_arg1,
  344. scratch->next_addr,
  345. scratch->next_mode, FALSE);
  346. }
  347. static atomic_t coldboot_lottery = ATOMIC_INITIALIZER(0);
  348. /**
  349. * Initialize OpenSBI library for current HART and jump to next
  350. * booting stage.
  351. *
  352. * The function expects following:
  353. * 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
  354. * 2. Stack pointer (SP) is setup for current HART
  355. * 3. Interrupts are disabled in MSTATUS CSR
  356. * 4. All interrupts are disabled in MIE CSR
  357. *
  358. * @param scratch pointer to sbi_scratch of current HART
  359. */
  360. void __noreturn sbi_init(struct sbi_scratch *scratch)
  361. {
  362. bool next_mode_supported = FALSE;
  363. bool coldboot = FALSE;
  364. u32 hartid = current_hartid();
  365. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  366. if ((SBI_HARTMASK_MAX_BITS <= hartid) ||
  367. sbi_platform_hart_invalid(plat, hartid))
  368. sbi_hart_hang();
  369. switch (scratch->next_mode) {
  370. case PRV_M:
  371. next_mode_supported = TRUE;
  372. break;
  373. case PRV_S:
  374. if (misa_extension('S'))
  375. next_mode_supported = TRUE;
  376. break;
  377. case PRV_U:
  378. if (misa_extension('U'))
  379. next_mode_supported = TRUE;
  380. break;
  381. default:
  382. sbi_hart_hang();
  383. }
  384. /*
  385. * Only the HART supporting privilege mode specified in the
  386. * scratch->next_mode should be allowed to become the coldboot
  387. * HART because the coldboot HART will be directly jumping to
  388. * the next booting stage.
  389. *
  390. * We use a lottery mechanism to select coldboot HART among
  391. * HARTs which satisfy above condition.
  392. */
  393. if (next_mode_supported && atomic_xchg(&coldboot_lottery, 1) == 0)
  394. coldboot = TRUE;
  395. if (coldboot)
  396. init_coldboot(scratch, hartid);
  397. else
  398. init_warmboot(scratch, hartid);
  399. }
  400. unsigned long sbi_init_count(u32 hartid)
  401. {
  402. struct sbi_scratch *scratch;
  403. unsigned long *init_count;
  404. if (!init_count_offset)
  405. return 0;
  406. scratch = sbi_hartid_to_scratch(hartid);
  407. if (!scratch)
  408. return 0;
  409. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  410. return *init_count;
  411. }
  412. /**
  413. * Exit OpenSBI library for current HART and stop HART
  414. *
  415. * The function expects following:
  416. * 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
  417. * 2. Stack pointer (SP) is setup for current HART
  418. *
  419. * @param scratch pointer to sbi_scratch of current HART
  420. */
  421. void __noreturn sbi_exit(struct sbi_scratch *scratch)
  422. {
  423. u32 hartid = current_hartid();
  424. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  425. if (sbi_platform_hart_invalid(plat, hartid))
  426. sbi_hart_hang();
  427. sbi_platform_early_exit(plat);
  428. sbi_pmu_exit(scratch);
  429. sbi_timer_exit(scratch);
  430. sbi_ipi_exit(scratch);
  431. sbi_platform_irqchip_exit(plat);
  432. sbi_platform_final_exit(plat);
  433. sbi_hsm_exit(scratch);
  434. }