sbi_trap.c 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348
  1. /*
  2. * SPDX-License-Identifier: BSD-2-Clause
  3. *
  4. * Copyright (c) 2019 Western Digital Corporation or its affiliates.
  5. *
  6. * Authors:
  7. * Anup Patel <anup.patel@wdc.com>
  8. */
  9. #include <sbi/riscv_asm.h>
  10. #include <sbi/riscv_encoding.h>
  11. #include <sbi/sbi_bitops.h>
  12. #include <sbi/sbi_console.h>
  13. #include <sbi/sbi_ecall.h>
  14. #include <sbi/sbi_error.h>
  15. #include <sbi/sbi_hart.h>
  16. #include <sbi/sbi_illegal_insn.h>
  17. #include <sbi/sbi_ipi.h>
  18. #include <sbi/sbi_irqchip.h>
  19. #include <sbi/sbi_misaligned_ldst.h>
  20. #include <sbi/sbi_pmu.h>
  21. #include <sbi/sbi_scratch.h>
  22. #include <sbi/sbi_timer.h>
  23. #include <sbi/sbi_trap.h>
  24. static void __noreturn sbi_trap_error(const char *msg, int rc,
  25. ulong mcause, ulong mtval, ulong mtval2,
  26. ulong mtinst, struct sbi_trap_regs *regs)
  27. {
  28. u32 hartid = current_hartid();
  29. sbi_printf("%s: hart%d: %s (error %d)\n", __func__, hartid, msg, rc);
  30. sbi_printf("%s: hart%d: mcause=0x%" PRILX " mtval=0x%" PRILX "\n",
  31. __func__, hartid, mcause, mtval);
  32. if (misa_extension('H')) {
  33. sbi_printf("%s: hart%d: mtval2=0x%" PRILX
  34. " mtinst=0x%" PRILX "\n",
  35. __func__, hartid, mtval2, mtinst);
  36. }
  37. sbi_printf("%s: hart%d: mepc=0x%" PRILX " mstatus=0x%" PRILX "\n",
  38. __func__, hartid, regs->mepc, regs->mstatus);
  39. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  40. hartid, "ra", regs->ra, "sp", regs->sp);
  41. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  42. hartid, "gp", regs->gp, "tp", regs->tp);
  43. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  44. hartid, "s0", regs->s0, "s1", regs->s1);
  45. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  46. hartid, "a0", regs->a0, "a1", regs->a1);
  47. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  48. hartid, "a2", regs->a2, "a3", regs->a3);
  49. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  50. hartid, "a4", regs->a4, "a5", regs->a5);
  51. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  52. hartid, "a6", regs->a6, "a7", regs->a7);
  53. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  54. hartid, "s2", regs->s2, "s3", regs->s3);
  55. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  56. hartid, "s4", regs->s4, "s5", regs->s5);
  57. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  58. hartid, "s6", regs->s6, "s7", regs->s7);
  59. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  60. hartid, "s8", regs->s8, "s9", regs->s9);
  61. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  62. hartid, "s10", regs->s10, "s11", regs->s11);
  63. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  64. hartid, "t0", regs->t0, "t1", regs->t1);
  65. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  66. hartid, "t2", regs->t2, "t3", regs->t3);
  67. sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
  68. hartid, "t4", regs->t4, "t5", regs->t5);
  69. sbi_printf("%s: hart%d: %s=0x%" PRILX "\n", __func__, hartid, "t6",
  70. regs->t6);
  71. sbi_hart_hang();
  72. }
  73. /**
  74. * Redirect trap to lower privledge mode (S-mode or U-mode)
  75. *
  76. * @param regs pointer to register state
  77. * @param trap pointer to trap details
  78. *
  79. * @return 0 on success and negative error code on failure
  80. */
  81. int sbi_trap_redirect(struct sbi_trap_regs *regs,
  82. struct sbi_trap_info *trap)
  83. {
  84. ulong hstatus, vsstatus, prev_mode;
  85. #if __riscv_xlen == 32
  86. bool prev_virt = (regs->mstatusH & MSTATUSH_MPV) ? true : false;
  87. #else
  88. bool prev_virt = (regs->mstatus & MSTATUS_MPV) ? true : false;
  89. #endif
  90. /* By default, we redirect to HS-mode */
  91. bool next_virt = false;
  92. /* Sanity check on previous mode */
  93. prev_mode = (regs->mstatus & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT;
  94. if (prev_mode != PRV_S && prev_mode != PRV_U)
  95. return SBI_ENOTSUPP;
  96. /* If exceptions came from VS/VU-mode, redirect to VS-mode if
  97. * delegated in hedeleg
  98. */
  99. if (misa_extension('H') && prev_virt) {
  100. if ((trap->cause < __riscv_xlen) &&
  101. (csr_read(CSR_HEDELEG) & BIT(trap->cause))) {
  102. next_virt = true;
  103. }
  104. }
  105. /* Update MSTATUS MPV bits */
  106. #if __riscv_xlen == 32
  107. regs->mstatusH &= ~MSTATUSH_MPV;
  108. regs->mstatusH |= (next_virt) ? MSTATUSH_MPV : 0UL;
  109. #else
  110. regs->mstatus &= ~MSTATUS_MPV;
  111. regs->mstatus |= (next_virt) ? MSTATUS_MPV : 0UL;
  112. #endif
  113. /* Update hypervisor CSRs if going to HS-mode */
  114. if (misa_extension('H') && !next_virt) {
  115. hstatus = csr_read(CSR_HSTATUS);
  116. if (prev_virt) {
  117. /* hstatus.SPVP is only updated if coming from VS/VU-mode */
  118. hstatus &= ~HSTATUS_SPVP;
  119. hstatus |= (prev_mode == PRV_S) ? HSTATUS_SPVP : 0;
  120. }
  121. hstatus &= ~HSTATUS_SPV;
  122. hstatus |= (prev_virt) ? HSTATUS_SPV : 0;
  123. hstatus &= ~HSTATUS_GVA;
  124. hstatus |= (trap->gva) ? HSTATUS_GVA : 0;
  125. csr_write(CSR_HSTATUS, hstatus);
  126. csr_write(CSR_HTVAL, trap->tval2);
  127. csr_write(CSR_HTINST, trap->tinst);
  128. }
  129. /* Update exception related CSRs */
  130. if (next_virt) {
  131. /* Update VS-mode exception info */
  132. csr_write(CSR_VSTVAL, trap->tval);
  133. csr_write(CSR_VSEPC, trap->epc);
  134. csr_write(CSR_VSCAUSE, trap->cause);
  135. /* Set MEPC to VS-mode exception vector base */
  136. regs->mepc = csr_read(CSR_VSTVEC);
  137. /* Set MPP to VS-mode */
  138. regs->mstatus &= ~MSTATUS_MPP;
  139. regs->mstatus |= (PRV_S << MSTATUS_MPP_SHIFT);
  140. /* Get VS-mode SSTATUS CSR */
  141. vsstatus = csr_read(CSR_VSSTATUS);
  142. /* Set SPP for VS-mode */
  143. vsstatus &= ~SSTATUS_SPP;
  144. if (prev_mode == PRV_S)
  145. vsstatus |= (1UL << SSTATUS_SPP_SHIFT);
  146. /* Set SPIE for VS-mode */
  147. vsstatus &= ~SSTATUS_SPIE;
  148. if (vsstatus & SSTATUS_SIE)
  149. vsstatus |= (1UL << SSTATUS_SPIE_SHIFT);
  150. /* Clear SIE for VS-mode */
  151. vsstatus &= ~SSTATUS_SIE;
  152. /* Update VS-mode SSTATUS CSR */
  153. csr_write(CSR_VSSTATUS, vsstatus);
  154. } else {
  155. /* Update S-mode exception info */
  156. csr_write(CSR_STVAL, trap->tval);
  157. csr_write(CSR_SEPC, trap->epc);
  158. csr_write(CSR_SCAUSE, trap->cause);
  159. /* Set MEPC to S-mode exception vector base */
  160. regs->mepc = csr_read(CSR_STVEC);
  161. /* Set MPP to S-mode */
  162. regs->mstatus &= ~MSTATUS_MPP;
  163. regs->mstatus |= (PRV_S << MSTATUS_MPP_SHIFT);
  164. /* Set SPP for S-mode */
  165. regs->mstatus &= ~MSTATUS_SPP;
  166. if (prev_mode == PRV_S)
  167. regs->mstatus |= (1UL << MSTATUS_SPP_SHIFT);
  168. /* Set SPIE for S-mode */
  169. regs->mstatus &= ~MSTATUS_SPIE;
  170. if (regs->mstatus & MSTATUS_SIE)
  171. regs->mstatus |= (1UL << MSTATUS_SPIE_SHIFT);
  172. /* Clear SIE for S-mode */
  173. regs->mstatus &= ~MSTATUS_SIE;
  174. }
  175. return 0;
  176. }
  177. static int sbi_trap_nonaia_irq(struct sbi_trap_regs *regs, ulong mcause)
  178. {
  179. mcause &= ~(1UL << (__riscv_xlen - 1));
  180. switch (mcause) {
  181. case IRQ_M_TIMER:
  182. sbi_timer_process();
  183. break;
  184. case IRQ_M_SOFT:
  185. sbi_ipi_process();
  186. break;
  187. case IRQ_M_EXT:
  188. return sbi_irqchip_process(regs);
  189. default:
  190. return SBI_ENOENT;
  191. };
  192. return 0;
  193. }
  194. static int sbi_trap_aia_irq(struct sbi_trap_regs *regs, ulong mcause)
  195. {
  196. int rc;
  197. unsigned long mtopi;
  198. while ((mtopi = csr_read(CSR_MTOPI))) {
  199. mtopi = mtopi >> TOPI_IID_SHIFT;
  200. switch (mtopi) {
  201. case IRQ_M_TIMER:
  202. sbi_timer_process();
  203. break;
  204. case IRQ_M_SOFT:
  205. sbi_ipi_process();
  206. break;
  207. case IRQ_M_EXT:
  208. rc = sbi_irqchip_process(regs);
  209. if (rc)
  210. return rc;
  211. break;
  212. default:
  213. return SBI_ENOENT;
  214. }
  215. }
  216. return 0;
  217. }
  218. /**
  219. * Handle trap/interrupt
  220. *
  221. * This function is called by firmware linked to OpenSBI
  222. * library for handling trap/interrupt. It expects the
  223. * following:
  224. * 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
  225. * 2. The 'mcause' CSR is having exception/interrupt cause
  226. * 3. The 'mtval' CSR is having additional trap information
  227. * 4. The 'mtval2' CSR is having additional trap information
  228. * 5. The 'mtinst' CSR is having decoded trap instruction
  229. * 6. Stack pointer (SP) is setup for current HART
  230. * 7. Interrupts are disabled in MSTATUS CSR
  231. *
  232. * @param regs pointer to register state
  233. */
  234. struct sbi_trap_regs *sbi_trap_handler(struct sbi_trap_regs *regs)
  235. {
  236. int rc = SBI_ENOTSUPP;
  237. const char *msg = "trap handler failed";
  238. ulong mcause = csr_read(CSR_MCAUSE);
  239. ulong mtval = csr_read(CSR_MTVAL), mtval2 = 0, mtinst = 0;
  240. struct sbi_trap_info trap;
  241. if (misa_extension('H')) {
  242. mtval2 = csr_read(CSR_MTVAL2);
  243. mtinst = csr_read(CSR_MTINST);
  244. }
  245. if (mcause & (1UL << (__riscv_xlen - 1))) {
  246. if (sbi_hart_has_extension(sbi_scratch_thishart_ptr(),
  247. SBI_HART_EXT_SMAIA))
  248. rc = sbi_trap_aia_irq(regs, mcause);
  249. else
  250. rc = sbi_trap_nonaia_irq(regs, mcause);
  251. if (rc) {
  252. msg = "unhandled local interrupt";
  253. goto trap_error;
  254. }
  255. return regs;
  256. }
  257. switch (mcause) {
  258. case CAUSE_ILLEGAL_INSTRUCTION:
  259. rc = sbi_illegal_insn_handler(mtval, regs);
  260. msg = "illegal instruction handler failed";
  261. break;
  262. case CAUSE_MISALIGNED_LOAD:
  263. rc = sbi_misaligned_load_handler(mtval, mtval2, mtinst, regs);
  264. msg = "misaligned load handler failed";
  265. break;
  266. case CAUSE_MISALIGNED_STORE:
  267. rc = sbi_misaligned_store_handler(mtval, mtval2, mtinst, regs);
  268. msg = "misaligned store handler failed";
  269. break;
  270. case CAUSE_SUPERVISOR_ECALL:
  271. case CAUSE_MACHINE_ECALL:
  272. rc = sbi_ecall_handler(regs);
  273. msg = "ecall handler failed";
  274. break;
  275. case CAUSE_LOAD_ACCESS:
  276. case CAUSE_STORE_ACCESS:
  277. sbi_pmu_ctr_incr_fw(mcause == CAUSE_LOAD_ACCESS ?
  278. SBI_PMU_FW_ACCESS_LOAD : SBI_PMU_FW_ACCESS_STORE);
  279. /* fallthrough */
  280. default:
  281. /* If the trap came from S or U mode, redirect it there */
  282. trap.epc = regs->mepc;
  283. trap.cause = mcause;
  284. trap.tval = mtval;
  285. trap.tval2 = mtval2;
  286. trap.tinst = mtinst;
  287. trap.gva = sbi_regs_gva(regs);
  288. rc = sbi_trap_redirect(regs, &trap);
  289. break;
  290. };
  291. trap_error:
  292. if (rc)
  293. sbi_trap_error(msg, rc, mcause, mtval, mtval2, mtinst, regs);
  294. return regs;
  295. }
  296. typedef void (*trap_exit_t)(const struct sbi_trap_regs *regs);
  297. /**
  298. * Exit trap/interrupt handling
  299. *
  300. * This function is called by non-firmware code to abruptly exit
  301. * trap/interrupt handling and resume execution at context pointed
  302. * by given register state.
  303. *
  304. * @param regs pointer to register state
  305. */
  306. void __noreturn sbi_trap_exit(const struct sbi_trap_regs *regs)
  307. {
  308. struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
  309. ((trap_exit_t)scratch->trap_exit)(regs);
  310. __builtin_unreachable();
  311. }