traps_misaligned.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2020 Western Digital Corporation or its affiliates.
  4. */
  5. #include <linux/kernel.h>
  6. #include <linux/init.h>
  7. #include <linux/mm.h>
  8. #include <linux/module.h>
  9. #include <linux/irq.h>
  10. #include <asm/processor.h>
  11. #include <asm/ptrace.h>
  12. #include <asm/csr.h>
  13. #define INSN_MATCH_LB 0x3
  14. #define INSN_MASK_LB 0x707f
  15. #define INSN_MATCH_LH 0x1003
  16. #define INSN_MASK_LH 0x707f
  17. #define INSN_MATCH_LW 0x2003
  18. #define INSN_MASK_LW 0x707f
  19. #define INSN_MATCH_LD 0x3003
  20. #define INSN_MASK_LD 0x707f
  21. #define INSN_MATCH_LBU 0x4003
  22. #define INSN_MASK_LBU 0x707f
  23. #define INSN_MATCH_LHU 0x5003
  24. #define INSN_MASK_LHU 0x707f
  25. #define INSN_MATCH_LWU 0x6003
  26. #define INSN_MASK_LWU 0x707f
  27. #define INSN_MATCH_SB 0x23
  28. #define INSN_MASK_SB 0x707f
  29. #define INSN_MATCH_SH 0x1023
  30. #define INSN_MASK_SH 0x707f
  31. #define INSN_MATCH_SW 0x2023
  32. #define INSN_MASK_SW 0x707f
  33. #define INSN_MATCH_SD 0x3023
  34. #define INSN_MASK_SD 0x707f
  35. #define INSN_MATCH_FLW 0x2007
  36. #define INSN_MASK_FLW 0x707f
  37. #define INSN_MATCH_FLD 0x3007
  38. #define INSN_MASK_FLD 0x707f
  39. #define INSN_MATCH_FLQ 0x4007
  40. #define INSN_MASK_FLQ 0x707f
  41. #define INSN_MATCH_FSW 0x2027
  42. #define INSN_MASK_FSW 0x707f
  43. #define INSN_MATCH_FSD 0x3027
  44. #define INSN_MASK_FSD 0x707f
  45. #define INSN_MATCH_FSQ 0x4027
  46. #define INSN_MASK_FSQ 0x707f
  47. #define INSN_MATCH_C_LD 0x6000
  48. #define INSN_MASK_C_LD 0xe003
  49. #define INSN_MATCH_C_SD 0xe000
  50. #define INSN_MASK_C_SD 0xe003
  51. #define INSN_MATCH_C_LW 0x4000
  52. #define INSN_MASK_C_LW 0xe003
  53. #define INSN_MATCH_C_SW 0xc000
  54. #define INSN_MASK_C_SW 0xe003
  55. #define INSN_MATCH_C_LDSP 0x6002
  56. #define INSN_MASK_C_LDSP 0xe003
  57. #define INSN_MATCH_C_SDSP 0xe002
  58. #define INSN_MASK_C_SDSP 0xe003
  59. #define INSN_MATCH_C_LWSP 0x4002
  60. #define INSN_MASK_C_LWSP 0xe003
  61. #define INSN_MATCH_C_SWSP 0xc002
  62. #define INSN_MASK_C_SWSP 0xe003
  63. #define INSN_MATCH_C_FLD 0x2000
  64. #define INSN_MASK_C_FLD 0xe003
  65. #define INSN_MATCH_C_FLW 0x6000
  66. #define INSN_MASK_C_FLW 0xe003
  67. #define INSN_MATCH_C_FSD 0xa000
  68. #define INSN_MASK_C_FSD 0xe003
  69. #define INSN_MATCH_C_FSW 0xe000
  70. #define INSN_MASK_C_FSW 0xe003
  71. #define INSN_MATCH_C_FLDSP 0x2002
  72. #define INSN_MASK_C_FLDSP 0xe003
  73. #define INSN_MATCH_C_FSDSP 0xa002
  74. #define INSN_MASK_C_FSDSP 0xe003
  75. #define INSN_MATCH_C_FLWSP 0x6002
  76. #define INSN_MASK_C_FLWSP 0xe003
  77. #define INSN_MATCH_C_FSWSP 0xe002
  78. #define INSN_MASK_C_FSWSP 0xe003
  79. #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4)
  80. #if defined(CONFIG_64BIT)
  81. #define LOG_REGBYTES 3
  82. #define XLEN 64
  83. #else
  84. #define LOG_REGBYTES 2
  85. #define XLEN 32
  86. #endif
  87. #define REGBYTES (1 << LOG_REGBYTES)
  88. #define XLEN_MINUS_16 ((XLEN) - 16)
  89. #define SH_RD 7
  90. #define SH_RS1 15
  91. #define SH_RS2 20
  92. #define SH_RS2C 2
  93. #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
  94. #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
  95. (RV_X(x, 10, 3) << 3) | \
  96. (RV_X(x, 5, 1) << 6))
  97. #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \
  98. (RV_X(x, 5, 2) << 6))
  99. #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \
  100. (RV_X(x, 12, 1) << 5) | \
  101. (RV_X(x, 2, 2) << 6))
  102. #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \
  103. (RV_X(x, 12, 1) << 5) | \
  104. (RV_X(x, 2, 3) << 6))
  105. #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \
  106. (RV_X(x, 7, 2) << 6))
  107. #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \
  108. (RV_X(x, 7, 3) << 6))
  109. #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3))
  110. #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3))
  111. #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5)
  112. #define SHIFT_RIGHT(x, y) \
  113. ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
  114. #define REG_MASK \
  115. ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
  116. #define REG_OFFSET(insn, pos) \
  117. (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
  118. #define REG_PTR(insn, pos, regs) \
  119. (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
  120. #define GET_RM(insn) (((insn) >> 12) & 7)
  121. #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs))
  122. #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs))
  123. #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs))
  124. #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs))
  125. #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs))
  126. #define GET_SP(regs) (*REG_PTR(2, 0, regs))
  127. #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val))
  128. #define IMM_I(insn) ((s32)(insn) >> 20)
  129. #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \
  130. (s32)(((insn) >> 7) & 0x1f))
  131. #define MASK_FUNCT3 0x7000
  132. #define GET_PRECISION(insn) (((insn) >> 25) & 3)
  133. #define GET_RM(insn) (((insn) >> 12) & 7)
  134. #define PRECISION_S 0
  135. #define PRECISION_D 1
  136. #define STR(x) XSTR(x)
  137. #define XSTR(x) #x
  138. #define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type, insn) \
  139. static inline type load_##type(const type *addr) \
  140. { \
  141. type val; \
  142. asm (#insn " %0, %1" \
  143. : "=&r" (val) : "m" (*addr)); \
  144. return val; \
  145. }
  146. #define DECLARE_UNPRIVILEGED_STORE_FUNCTION(type, insn) \
  147. static inline void store_##type(type *addr, type val) \
  148. { \
  149. asm volatile (#insn " %0, %1\n" \
  150. : : "r" (val), "m" (*addr)); \
  151. }
  152. DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u8, lbu)
  153. DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u16, lhu)
  154. DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s8, lb)
  155. DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s16, lh)
  156. DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s32, lw)
  157. DECLARE_UNPRIVILEGED_STORE_FUNCTION(u8, sb)
  158. DECLARE_UNPRIVILEGED_STORE_FUNCTION(u16, sh)
  159. DECLARE_UNPRIVILEGED_STORE_FUNCTION(u32, sw)
  160. #if defined(CONFIG_64BIT)
  161. DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u32, lwu)
  162. DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u64, ld)
  163. DECLARE_UNPRIVILEGED_STORE_FUNCTION(u64, sd)
  164. DECLARE_UNPRIVILEGED_LOAD_FUNCTION(ulong, ld)
  165. #else
  166. DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u32, lw)
  167. DECLARE_UNPRIVILEGED_LOAD_FUNCTION(ulong, lw)
  168. static inline u64 load_u64(const u64 *addr)
  169. {
  170. return load_u32((u32 *)addr)
  171. + ((u64)load_u32((u32 *)addr + 1) << 32);
  172. }
  173. static inline void store_u64(u64 *addr, u64 val)
  174. {
  175. store_u32((u32 *)addr, val);
  176. store_u32((u32 *)addr + 1, val >> 32);
  177. }
  178. #endif
  179. static inline ulong get_insn(ulong mepc)
  180. {
  181. register ulong __mepc asm ("a2") = mepc;
  182. ulong val, rvc_mask = 3, tmp;
  183. asm ("and %[tmp], %[addr], 2\n"
  184. "bnez %[tmp], 1f\n"
  185. #if defined(CONFIG_64BIT)
  186. STR(LWU) " %[insn], (%[addr])\n"
  187. #else
  188. STR(LW) " %[insn], (%[addr])\n"
  189. #endif
  190. "and %[tmp], %[insn], %[rvc_mask]\n"
  191. "beq %[tmp], %[rvc_mask], 2f\n"
  192. "sll %[insn], %[insn], %[xlen_minus_16]\n"
  193. "srl %[insn], %[insn], %[xlen_minus_16]\n"
  194. "j 2f\n"
  195. "1:\n"
  196. "lhu %[insn], (%[addr])\n"
  197. "and %[tmp], %[insn], %[rvc_mask]\n"
  198. "bne %[tmp], %[rvc_mask], 2f\n"
  199. "lhu %[tmp], 2(%[addr])\n"
  200. "sll %[tmp], %[tmp], 16\n"
  201. "add %[insn], %[insn], %[tmp]\n"
  202. "2:"
  203. : [insn] "=&r" (val), [tmp] "=&r" (tmp)
  204. : [addr] "r" (__mepc), [rvc_mask] "r" (rvc_mask),
  205. [xlen_minus_16] "i" (XLEN_MINUS_16));
  206. return val;
  207. }
  208. union reg_data {
  209. u8 data_bytes[8];
  210. ulong data_ulong;
  211. u64 data_u64;
  212. };
  213. int handle_misaligned_load(struct pt_regs *regs)
  214. {
  215. union reg_data val;
  216. unsigned long epc = regs->epc;
  217. unsigned long insn = get_insn(epc);
  218. unsigned long addr = csr_read(mtval);
  219. int i, fp = 0, shift = 0, len = 0;
  220. regs->epc = 0;
  221. if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
  222. len = 4;
  223. shift = 8 * (sizeof(unsigned long) - len);
  224. #if defined(CONFIG_64BIT)
  225. } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
  226. len = 8;
  227. shift = 8 * (sizeof(unsigned long) - len);
  228. } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
  229. len = 4;
  230. #endif
  231. } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
  232. fp = 1;
  233. len = 8;
  234. } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
  235. fp = 1;
  236. len = 4;
  237. } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
  238. len = 2;
  239. shift = 8 * (sizeof(unsigned long) - len);
  240. } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
  241. len = 2;
  242. #if defined(CONFIG_64BIT)
  243. } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
  244. len = 8;
  245. shift = 8 * (sizeof(unsigned long) - len);
  246. insn = RVC_RS2S(insn) << SH_RD;
  247. } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
  248. ((insn >> SH_RD) & 0x1f)) {
  249. len = 8;
  250. shift = 8 * (sizeof(unsigned long) - len);
  251. #endif
  252. } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
  253. len = 4;
  254. shift = 8 * (sizeof(unsigned long) - len);
  255. insn = RVC_RS2S(insn) << SH_RD;
  256. } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
  257. ((insn >> SH_RD) & 0x1f)) {
  258. len = 4;
  259. shift = 8 * (sizeof(unsigned long) - len);
  260. } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
  261. fp = 1;
  262. len = 8;
  263. insn = RVC_RS2S(insn) << SH_RD;
  264. } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
  265. fp = 1;
  266. len = 8;
  267. #if defined(CONFIG_32BIT)
  268. } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
  269. fp = 1;
  270. len = 4;
  271. insn = RVC_RS2S(insn) << SH_RD;
  272. } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
  273. fp = 1;
  274. len = 4;
  275. #endif
  276. } else {
  277. regs->epc = epc;
  278. return -1;
  279. }
  280. val.data_u64 = 0;
  281. for (i = 0; i < len; i++)
  282. val.data_bytes[i] = load_u8((void *)(addr + i));
  283. if (fp)
  284. return -1;
  285. SET_RD(insn, regs, val.data_ulong << shift >> shift);
  286. regs->epc = epc + INSN_LEN(insn);
  287. return 0;
  288. }
  289. int handle_misaligned_store(struct pt_regs *regs)
  290. {
  291. union reg_data val;
  292. unsigned long epc = regs->epc;
  293. unsigned long insn = get_insn(epc);
  294. unsigned long addr = csr_read(mtval);
  295. int i, len = 0;
  296. regs->epc = 0;
  297. val.data_ulong = GET_RS2(insn, regs);
  298. if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
  299. len = 4;
  300. #if defined(CONFIG_64BIT)
  301. } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
  302. len = 8;
  303. #endif
  304. } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
  305. len = 2;
  306. #if defined(CONFIG_64BIT)
  307. } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
  308. len = 8;
  309. val.data_ulong = GET_RS2S(insn, regs);
  310. } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP &&
  311. ((insn >> SH_RD) & 0x1f)) {
  312. len = 8;
  313. val.data_ulong = GET_RS2C(insn, regs);
  314. #endif
  315. } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
  316. len = 4;
  317. val.data_ulong = GET_RS2S(insn, regs);
  318. } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP &&
  319. ((insn >> SH_RD) & 0x1f)) {
  320. len = 4;
  321. val.data_ulong = GET_RS2C(insn, regs);
  322. } else {
  323. regs->epc = epc;
  324. return -1;
  325. }
  326. for (i = 0; i < len; i++)
  327. store_u8((void *)(addr + i), val.data_bytes[i]);
  328. regs->epc = epc + INSN_LEN(insn);
  329. return 0;
  330. }