sbi_init.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530
  1. /*
  2. * SPDX-License-Identifier: BSD-2-Clause
  3. *
  4. * Copyright (c) 2019 Western Digital Corporation or its affiliates.
  5. *
  6. * Authors:
  7. * Anup Patel <anup.patel@wdc.com>
  8. */
  9. #include <sbi/riscv_asm.h>
  10. #include <sbi/riscv_atomic.h>
  11. #include <sbi/riscv_barrier.h>
  12. #include <sbi/riscv_locks.h>
  13. #include <sbi/sbi_console.h>
  14. #include <sbi/sbi_domain.h>
  15. #include <sbi/sbi_ecall.h>
  16. #include <sbi/sbi_hart.h>
  17. #include <sbi/sbi_hartmask.h>
  18. #include <sbi/sbi_hsm.h>
  19. #include <sbi/sbi_ipi.h>
  20. #include <sbi/sbi_platform.h>
  21. #include <sbi/sbi_system.h>
  22. #include <sbi/sbi_string.h>
  23. #include <sbi/sbi_timer.h>
  24. #include <sbi/sbi_tlb.h>
  25. #include <sbi/sbi_version.h>
  26. #define BANNER \
  27. " ____ _____ ____ _____\n" \
  28. " / __ \\ / ____| _ \\_ _|\n" \
  29. " | | | |_ __ ___ _ __ | (___ | |_) || |\n" \
  30. " | | | | '_ \\ / _ \\ '_ \\ \\___ \\| _ < | |\n" \
  31. " | |__| | |_) | __/ | | |____) | |_) || |_\n" \
  32. " \\____/| .__/ \\___|_| |_|_____/|____/_____|\n" \
  33. " | |\n" \
  34. " |_|\n\n"
  35. static void sbi_boot_print_banner(struct sbi_scratch *scratch)
  36. {
  37. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  38. return;
  39. #ifdef OPENSBI_VERSION_GIT
  40. sbi_printf("\nOpenSBI %s\n", OPENSBI_VERSION_GIT);
  41. #else
  42. sbi_printf("\nOpenSBI v%d.%d\n", OPENSBI_VERSION_MAJOR,
  43. OPENSBI_VERSION_MINOR);
  44. #endif
  45. sbi_printf(BANNER);
  46. }
  47. static void sbi_boot_print_general(struct sbi_scratch *scratch)
  48. {
  49. char str[128];
  50. const struct sbi_hsm_device *hdev;
  51. const struct sbi_ipi_device *idev;
  52. const struct sbi_timer_device *tdev;
  53. const struct sbi_console_device *cdev;
  54. const struct sbi_system_reset_device *srdev;
  55. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  56. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  57. return;
  58. /* Platform details */
  59. sbi_printf("Platform Name : %s\n",
  60. sbi_platform_name(plat));
  61. sbi_platform_get_features_str(plat, str, sizeof(str));
  62. sbi_printf("Platform Features : %s\n", str);
  63. sbi_printf("Platform HART Count : %u\n",
  64. sbi_platform_hart_count(plat));
  65. idev = sbi_ipi_get_device();
  66. sbi_printf("Platform IPI Device : %s\n",
  67. (idev) ? idev->name : "---");
  68. tdev = sbi_timer_get_device();
  69. sbi_printf("Platform Timer Device : %s\n",
  70. (tdev) ? tdev->name : "---");
  71. cdev = sbi_console_get_device();
  72. sbi_printf("Platform Console Device : %s\n",
  73. (cdev) ? cdev->name : "---");
  74. hdev = sbi_hsm_get_device();
  75. sbi_printf("Platform HSM Device : %s\n",
  76. (hdev) ? hdev->name : "---");
  77. srdev = sbi_system_reset_get_device();
  78. sbi_printf("Platform SysReset Device : %s\n",
  79. (srdev) ? srdev->name : "---");
  80. /* Firmware details */
  81. sbi_printf("Firmware Base : 0x%lx\n", scratch->fw_start);
  82. sbi_printf("Firmware Size : %d KB\n",
  83. (u32)(scratch->fw_size / 1024));
  84. /* SBI details */
  85. sbi_printf("Runtime SBI Version : %d.%d\n",
  86. sbi_ecall_version_major(), sbi_ecall_version_minor());
  87. sbi_printf("\n");
  88. }
  89. static void sbi_boot_print_domains(struct sbi_scratch *scratch)
  90. {
  91. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  92. return;
  93. /* Domain details */
  94. sbi_domain_dump_all(" ");
  95. }
  96. static void sbi_boot_print_hart(struct sbi_scratch *scratch, u32 hartid)
  97. {
  98. int xlen;
  99. char str[128];
  100. const struct sbi_domain *dom = sbi_domain_thishart_ptr();
  101. if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
  102. return;
  103. /* Determine MISA XLEN and MISA string */
  104. xlen = misa_xlen();
  105. if (xlen < 1) {
  106. sbi_printf("Error %d getting MISA XLEN\n", xlen);
  107. sbi_hart_hang();
  108. }
  109. /* Boot HART details */
  110. sbi_printf("Boot HART ID : %u\n", hartid);
  111. sbi_printf("Boot HART Domain : %s\n", dom->name);
  112. misa_string(xlen, str, sizeof(str));
  113. sbi_printf("Boot HART ISA : %s\n", str);
  114. sbi_hart_get_features_str(scratch, str, sizeof(str));
  115. sbi_printf("Boot HART Features : %s\n", str);
  116. sbi_printf("Boot HART PMP Count : %d\n",
  117. sbi_hart_pmp_count(scratch));
  118. sbi_printf("Boot HART PMP Granularity : %lu\n",
  119. sbi_hart_pmp_granularity(scratch));
  120. sbi_printf("Boot HART PMP Address Bits: %d\n",
  121. sbi_hart_pmp_addrbits(scratch));
  122. sbi_printf("Boot HART MHPM Count : %d\n",
  123. sbi_hart_mhpm_count(scratch));
  124. sbi_printf("Boot HART MHPM Count : %d\n",
  125. sbi_hart_mhpm_count(scratch));
  126. sbi_hart_delegation_dump(scratch, "Boot HART ", " ");
  127. }
  128. static spinlock_t coldboot_lock = SPIN_LOCK_INITIALIZER;
  129. static struct sbi_hartmask coldboot_wait_hmask = { 0 };
  130. static unsigned long coldboot_done;
  131. static void wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
  132. {
  133. unsigned long saved_mie, cmip;
  134. /* Save MIE CSR */
  135. saved_mie = csr_read(CSR_MIE);
  136. /* Set MSIE bit to receive IPI */
  137. csr_set(CSR_MIE, MIP_MSIP);
  138. /* Acquire coldboot lock */
  139. spin_lock(&coldboot_lock);
  140. /* Mark current HART as waiting */
  141. sbi_hartmask_set_hart(hartid, &coldboot_wait_hmask);
  142. /* Release coldboot lock */
  143. spin_unlock(&coldboot_lock);
  144. /* Wait for coldboot to finish using WFI */
  145. while (!__smp_load_acquire(&coldboot_done)) {
  146. do {
  147. wfi();
  148. cmip = csr_read(CSR_MIP);
  149. } while (!(cmip & MIP_MSIP));
  150. };
  151. /* Acquire coldboot lock */
  152. spin_lock(&coldboot_lock);
  153. /* Unmark current HART as waiting */
  154. sbi_hartmask_clear_hart(hartid, &coldboot_wait_hmask);
  155. /* Release coldboot lock */
  156. spin_unlock(&coldboot_lock);
  157. /* Restore MIE CSR */
  158. csr_write(CSR_MIE, saved_mie);
  159. /*
  160. * The wait for coldboot is common for both warm startup and
  161. * warm resume path so clearing IPI here would result in losing
  162. * an IPI in warm resume path.
  163. *
  164. * Also, the sbi_platform_ipi_init() called from sbi_ipi_init()
  165. * will automatically clear IPI for current HART.
  166. */
  167. }
  168. static void wake_coldboot_harts(struct sbi_scratch *scratch, u32 hartid)
  169. {
  170. /* Mark coldboot done */
  171. __smp_store_release(&coldboot_done, 1);
  172. /* Acquire coldboot lock */
  173. spin_lock(&coldboot_lock);
  174. /* Send an IPI to all HARTs waiting for coldboot */
  175. for (int i = 0; i <= sbi_scratch_last_hartid(); i++) {
  176. if ((i != hartid) &&
  177. sbi_hartmask_test_hart(i, &coldboot_wait_hmask))
  178. sbi_ipi_raw_send(i);
  179. }
  180. /* Release coldboot lock */
  181. spin_unlock(&coldboot_lock);
  182. }
  183. static unsigned long init_count_offset;
  184. static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
  185. {
  186. int rc;
  187. unsigned long *init_count;
  188. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  189. /* Note: This has to be first thing in coldboot init sequence */
  190. rc = sbi_scratch_init(scratch);
  191. if (rc)
  192. sbi_hart_hang();
  193. /* Note: This has to be second thing in coldboot init sequence */
  194. rc = sbi_domain_init(scratch, hartid);
  195. if (rc)
  196. sbi_hart_hang();
  197. init_count_offset = sbi_scratch_alloc_offset(__SIZEOF_POINTER__,
  198. "INIT_COUNT");
  199. if (!init_count_offset)
  200. sbi_hart_hang();
  201. rc = sbi_hsm_init(scratch, hartid, TRUE);
  202. if (rc)
  203. sbi_hart_hang();
  204. rc = sbi_platform_early_init(plat, TRUE);
  205. if (rc)
  206. sbi_hart_hang();
  207. rc = sbi_hart_init(scratch, TRUE);
  208. if (rc)
  209. sbi_hart_hang();
  210. rc = sbi_console_init(scratch);
  211. if (rc)
  212. sbi_hart_hang();
  213. sbi_boot_print_banner(scratch);
  214. rc = sbi_platform_irqchip_init(plat, TRUE);
  215. if (rc) {
  216. sbi_printf("%s: platform irqchip init failed (error %d)\n",
  217. __func__, rc);
  218. sbi_hart_hang();
  219. }
  220. rc = sbi_ipi_init(scratch, TRUE);
  221. if (rc) {
  222. sbi_printf("%s: ipi init failed (error %d)\n", __func__, rc);
  223. sbi_hart_hang();
  224. }
  225. rc = sbi_tlb_init(scratch, TRUE);
  226. if (rc) {
  227. sbi_printf("%s: tlb init failed (error %d)\n", __func__, rc);
  228. sbi_hart_hang();
  229. }
  230. rc = sbi_timer_init(scratch, TRUE);
  231. if (rc) {
  232. sbi_printf("%s: timer init failed (error %d)\n", __func__, rc);
  233. sbi_hart_hang();
  234. }
  235. rc = sbi_ecall_init();
  236. if (rc) {
  237. sbi_printf("%s: ecall init failed (error %d)\n", __func__, rc);
  238. sbi_hart_hang();
  239. }
  240. sbi_boot_print_general(scratch);
  241. /*
  242. * Note: Finalize domains after HSM initialization so that we
  243. * can startup non-root domains.
  244. * Note: Finalize domains before HART PMP configuration so
  245. * that we use correct domain for configuring PMP.
  246. */
  247. rc = sbi_domain_finalize(scratch, hartid);
  248. if (rc) {
  249. sbi_printf("%s: domain finalize failed (error %d)\n",
  250. __func__, rc);
  251. sbi_hart_hang();
  252. }
  253. sbi_boot_print_domains(scratch);
  254. rc = sbi_hart_pmp_configure(scratch);
  255. if (rc) {
  256. sbi_printf("%s: PMP configure failed (error %d)\n",
  257. __func__, rc);
  258. sbi_hart_hang();
  259. }
  260. /*
  261. * Note: Platform final initialization should be last so that
  262. * it sees correct domain assignment and PMP configuration.
  263. */
  264. rc = sbi_platform_final_init(plat, TRUE);
  265. if (rc) {
  266. sbi_printf("%s: platform final init failed (error %d)\n",
  267. __func__, rc);
  268. sbi_hart_hang();
  269. }
  270. sbi_boot_print_hart(scratch, hartid);
  271. wake_coldboot_harts(scratch, hartid);
  272. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  273. (*init_count)++;
  274. sbi_hsm_prepare_next_jump(scratch, hartid);
  275. sbi_hart_switch_mode(hartid, scratch->next_arg1, scratch->next_addr,
  276. scratch->next_mode, FALSE);
  277. }
  278. static void init_warm_startup(struct sbi_scratch *scratch, u32 hartid)
  279. {
  280. int rc;
  281. unsigned long *init_count;
  282. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  283. if (!init_count_offset)
  284. sbi_hart_hang();
  285. rc = sbi_hsm_init(scratch, hartid, FALSE);
  286. if (rc)
  287. sbi_hart_hang();
  288. rc = sbi_platform_early_init(plat, FALSE);
  289. if (rc)
  290. sbi_hart_hang();
  291. rc = sbi_hart_init(scratch, FALSE);
  292. if (rc)
  293. sbi_hart_hang();
  294. rc = sbi_platform_irqchip_init(plat, FALSE);
  295. if (rc)
  296. sbi_hart_hang();
  297. rc = sbi_ipi_init(scratch, FALSE);
  298. if (rc)
  299. sbi_hart_hang();
  300. rc = sbi_tlb_init(scratch, FALSE);
  301. if (rc)
  302. sbi_hart_hang();
  303. rc = sbi_timer_init(scratch, FALSE);
  304. if (rc)
  305. sbi_hart_hang();
  306. rc = sbi_hart_pmp_configure(scratch);
  307. if (rc)
  308. sbi_hart_hang();
  309. rc = sbi_platform_final_init(plat, FALSE);
  310. if (rc)
  311. sbi_hart_hang();
  312. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  313. (*init_count)++;
  314. sbi_hsm_prepare_next_jump(scratch, hartid);
  315. }
  316. static void init_warm_resume(struct sbi_scratch *scratch)
  317. {
  318. int rc;
  319. sbi_hsm_hart_resume_start(scratch);
  320. rc = sbi_hart_reinit(scratch);
  321. if (rc)
  322. sbi_hart_hang();
  323. rc = sbi_hart_pmp_configure(scratch);
  324. if (rc)
  325. sbi_hart_hang();
  326. sbi_hsm_hart_resume_finish(scratch);
  327. }
  328. static void __noreturn init_warmboot(struct sbi_scratch *scratch, u32 hartid)
  329. {
  330. int hstate;
  331. wait_for_coldboot(scratch, hartid);
  332. hstate = sbi_hsm_hart_get_state(sbi_domain_thishart_ptr(), hartid);
  333. if (hstate < 0)
  334. sbi_hart_hang();
  335. if (hstate == SBI_HSM_STATE_SUSPENDED)
  336. init_warm_resume(scratch);
  337. else
  338. init_warm_startup(scratch, hartid);
  339. sbi_hart_switch_mode(hartid, scratch->next_arg1,
  340. scratch->next_addr,
  341. scratch->next_mode, FALSE);
  342. }
  343. static atomic_t coldboot_lottery = ATOMIC_INITIALIZER(0);
  344. /**
  345. * Initialize OpenSBI library for current HART and jump to next
  346. * booting stage.
  347. *
  348. * The function expects following:
  349. * 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
  350. * 2. Stack pointer (SP) is setup for current HART
  351. * 3. Interrupts are disabled in MSTATUS CSR
  352. * 4. All interrupts are disabled in MIE CSR
  353. *
  354. * @param scratch pointer to sbi_scratch of current HART
  355. */
  356. void __noreturn sbi_init(struct sbi_scratch *scratch)
  357. {
  358. bool next_mode_supported = FALSE;
  359. bool coldboot = FALSE;
  360. u32 hartid = current_hartid();
  361. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  362. if ((SBI_HARTMASK_MAX_BITS <= hartid) ||
  363. sbi_platform_hart_invalid(plat, hartid))
  364. sbi_hart_hang();
  365. switch (scratch->next_mode) {
  366. case PRV_M:
  367. next_mode_supported = TRUE;
  368. break;
  369. case PRV_S:
  370. if (misa_extension('S'))
  371. next_mode_supported = TRUE;
  372. break;
  373. case PRV_U:
  374. if (misa_extension('U'))
  375. next_mode_supported = TRUE;
  376. break;
  377. default:
  378. sbi_hart_hang();
  379. }
  380. /*
  381. * Only the HART supporting privilege mode specified in the
  382. * scratch->next_mode should be allowed to become the coldboot
  383. * HART because the coldboot HART will be directly jumping to
  384. * the next booting stage.
  385. *
  386. * We use a lottery mechanism to select coldboot HART among
  387. * HARTs which satisfy above condition.
  388. */
  389. if (next_mode_supported && atomic_xchg(&coldboot_lottery, 1) == 0)
  390. coldboot = TRUE;
  391. if (coldboot)
  392. init_coldboot(scratch, hartid);
  393. else
  394. init_warmboot(scratch, hartid);
  395. }
  396. unsigned long sbi_init_count(u32 hartid)
  397. {
  398. struct sbi_scratch *scratch;
  399. unsigned long *init_count;
  400. if (!init_count_offset)
  401. return 0;
  402. scratch = sbi_hartid_to_scratch(hartid);
  403. if (!scratch)
  404. return 0;
  405. init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
  406. return *init_count;
  407. }
  408. /**
  409. * Exit OpenSBI library for current HART and stop HART
  410. *
  411. * The function expects following:
  412. * 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
  413. * 2. Stack pointer (SP) is setup for current HART
  414. *
  415. * @param scratch pointer to sbi_scratch of current HART
  416. */
  417. void __noreturn sbi_exit(struct sbi_scratch *scratch)
  418. {
  419. u32 hartid = current_hartid();
  420. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  421. if (sbi_platform_hart_invalid(plat, hartid))
  422. sbi_hart_hang();
  423. sbi_platform_early_exit(plat);
  424. sbi_timer_exit(scratch);
  425. sbi_ipi_exit(scratch);
  426. sbi_platform_irqchip_exit(plat);
  427. sbi_platform_final_exit(plat);
  428. sbi_hsm_exit(scratch);
  429. }