module.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. *
  4. * Copyright (C) 2017 Zihao Yu
  5. */
  6. #include <linux/elf.h>
  7. #include <linux/err.h>
  8. #include <linux/errno.h>
  9. #include <linux/moduleloader.h>
  10. #include <linux/vmalloc.h>
  11. #include <linux/sizes.h>
  12. #include <linux/pgtable.h>
  13. #include <asm/sections.h>
  14. /*
  15. * The auipc+jalr instruction pair can reach any PC-relative offset
  16. * in the range [-2^31 - 2^11, 2^31 - 2^11)
  17. */
  18. static bool riscv_insn_valid_32bit_offset(ptrdiff_t val)
  19. {
  20. #ifdef CONFIG_32BIT
  21. return true;
  22. #else
  23. return (-(1L << 31) - (1L << 11)) <= val && val < ((1L << 31) - (1L << 11));
  24. #endif
  25. }
  26. static int apply_r_riscv_32_rela(struct module *me, u32 *location, Elf_Addr v)
  27. {
  28. if (v != (u32)v) {
  29. pr_err("%s: value %016llx out of range for 32-bit field\n",
  30. me->name, (long long)v);
  31. return -EINVAL;
  32. }
  33. *location = v;
  34. return 0;
  35. }
  36. static int apply_r_riscv_64_rela(struct module *me, u32 *location, Elf_Addr v)
  37. {
  38. *(u64 *)location = v;
  39. return 0;
  40. }
  41. static int apply_r_riscv_branch_rela(struct module *me, u32 *location,
  42. Elf_Addr v)
  43. {
  44. ptrdiff_t offset = (void *)v - (void *)location;
  45. u32 imm12 = (offset & 0x1000) << (31 - 12);
  46. u32 imm11 = (offset & 0x800) >> (11 - 7);
  47. u32 imm10_5 = (offset & 0x7e0) << (30 - 10);
  48. u32 imm4_1 = (offset & 0x1e) << (11 - 4);
  49. *location = (*location & 0x1fff07f) | imm12 | imm11 | imm10_5 | imm4_1;
  50. return 0;
  51. }
  52. static int apply_r_riscv_jal_rela(struct module *me, u32 *location,
  53. Elf_Addr v)
  54. {
  55. ptrdiff_t offset = (void *)v - (void *)location;
  56. u32 imm20 = (offset & 0x100000) << (31 - 20);
  57. u32 imm19_12 = (offset & 0xff000);
  58. u32 imm11 = (offset & 0x800) << (20 - 11);
  59. u32 imm10_1 = (offset & 0x7fe) << (30 - 10);
  60. *location = (*location & 0xfff) | imm20 | imm19_12 | imm11 | imm10_1;
  61. return 0;
  62. }
  63. static int apply_r_riscv_rcv_branch_rela(struct module *me, u32 *location,
  64. Elf_Addr v)
  65. {
  66. ptrdiff_t offset = (void *)v - (void *)location;
  67. u16 imm8 = (offset & 0x100) << (12 - 8);
  68. u16 imm7_6 = (offset & 0xc0) >> (6 - 5);
  69. u16 imm5 = (offset & 0x20) >> (5 - 2);
  70. u16 imm4_3 = (offset & 0x18) << (12 - 5);
  71. u16 imm2_1 = (offset & 0x6) << (12 - 10);
  72. *(u16 *)location = (*(u16 *)location & 0xe383) |
  73. imm8 | imm7_6 | imm5 | imm4_3 | imm2_1;
  74. return 0;
  75. }
  76. static int apply_r_riscv_rvc_jump_rela(struct module *me, u32 *location,
  77. Elf_Addr v)
  78. {
  79. ptrdiff_t offset = (void *)v - (void *)location;
  80. u16 imm11 = (offset & 0x800) << (12 - 11);
  81. u16 imm10 = (offset & 0x400) >> (10 - 8);
  82. u16 imm9_8 = (offset & 0x300) << (12 - 11);
  83. u16 imm7 = (offset & 0x80) >> (7 - 6);
  84. u16 imm6 = (offset & 0x40) << (12 - 11);
  85. u16 imm5 = (offset & 0x20) >> (5 - 2);
  86. u16 imm4 = (offset & 0x10) << (12 - 5);
  87. u16 imm3_1 = (offset & 0xe) << (12 - 10);
  88. *(u16 *)location = (*(u16 *)location & 0xe003) |
  89. imm11 | imm10 | imm9_8 | imm7 | imm6 | imm5 | imm4 | imm3_1;
  90. return 0;
  91. }
  92. static int apply_r_riscv_pcrel_hi20_rela(struct module *me, u32 *location,
  93. Elf_Addr v)
  94. {
  95. ptrdiff_t offset = (void *)v - (void *)location;
  96. s32 hi20;
  97. if (!riscv_insn_valid_32bit_offset(offset)) {
  98. pr_err(
  99. "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
  100. me->name, (long long)v, location);
  101. return -EINVAL;
  102. }
  103. hi20 = (offset + 0x800) & 0xfffff000;
  104. *location = (*location & 0xfff) | hi20;
  105. return 0;
  106. }
  107. static int apply_r_riscv_pcrel_lo12_i_rela(struct module *me, u32 *location,
  108. Elf_Addr v)
  109. {
  110. /*
  111. * v is the lo12 value to fill. It is calculated before calling this
  112. * handler.
  113. */
  114. *location = (*location & 0xfffff) | ((v & 0xfff) << 20);
  115. return 0;
  116. }
  117. static int apply_r_riscv_pcrel_lo12_s_rela(struct module *me, u32 *location,
  118. Elf_Addr v)
  119. {
  120. /*
  121. * v is the lo12 value to fill. It is calculated before calling this
  122. * handler.
  123. */
  124. u32 imm11_5 = (v & 0xfe0) << (31 - 11);
  125. u32 imm4_0 = (v & 0x1f) << (11 - 4);
  126. *location = (*location & 0x1fff07f) | imm11_5 | imm4_0;
  127. return 0;
  128. }
  129. static int apply_r_riscv_hi20_rela(struct module *me, u32 *location,
  130. Elf_Addr v)
  131. {
  132. s32 hi20;
  133. if (IS_ENABLED(CONFIG_CMODEL_MEDLOW)) {
  134. pr_err(
  135. "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
  136. me->name, (long long)v, location);
  137. return -EINVAL;
  138. }
  139. hi20 = ((s32)v + 0x800) & 0xfffff000;
  140. *location = (*location & 0xfff) | hi20;
  141. return 0;
  142. }
  143. static int apply_r_riscv_lo12_i_rela(struct module *me, u32 *location,
  144. Elf_Addr v)
  145. {
  146. /* Skip medlow checking because of filtering by HI20 already */
  147. s32 hi20 = ((s32)v + 0x800) & 0xfffff000;
  148. s32 lo12 = ((s32)v - hi20);
  149. *location = (*location & 0xfffff) | ((lo12 & 0xfff) << 20);
  150. return 0;
  151. }
  152. static int apply_r_riscv_lo12_s_rela(struct module *me, u32 *location,
  153. Elf_Addr v)
  154. {
  155. /* Skip medlow checking because of filtering by HI20 already */
  156. s32 hi20 = ((s32)v + 0x800) & 0xfffff000;
  157. s32 lo12 = ((s32)v - hi20);
  158. u32 imm11_5 = (lo12 & 0xfe0) << (31 - 11);
  159. u32 imm4_0 = (lo12 & 0x1f) << (11 - 4);
  160. *location = (*location & 0x1fff07f) | imm11_5 | imm4_0;
  161. return 0;
  162. }
  163. static int apply_r_riscv_got_hi20_rela(struct module *me, u32 *location,
  164. Elf_Addr v)
  165. {
  166. ptrdiff_t offset = (void *)v - (void *)location;
  167. s32 hi20;
  168. /* Always emit the got entry */
  169. if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) {
  170. offset = module_emit_got_entry(me, v);
  171. offset = (void *)offset - (void *)location;
  172. } else {
  173. pr_err(
  174. "%s: can not generate the GOT entry for symbol = %016llx from PC = %p\n",
  175. me->name, (long long)v, location);
  176. return -EINVAL;
  177. }
  178. hi20 = (offset + 0x800) & 0xfffff000;
  179. *location = (*location & 0xfff) | hi20;
  180. return 0;
  181. }
  182. static int apply_r_riscv_call_plt_rela(struct module *me, u32 *location,
  183. Elf_Addr v)
  184. {
  185. ptrdiff_t offset = (void *)v - (void *)location;
  186. u32 hi20, lo12;
  187. if (!riscv_insn_valid_32bit_offset(offset)) {
  188. /* Only emit the plt entry if offset over 32-bit range */
  189. if (IS_ENABLED(CONFIG_MODULE_SECTIONS)) {
  190. offset = module_emit_plt_entry(me, v);
  191. offset = (void *)offset - (void *)location;
  192. } else {
  193. pr_err(
  194. "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
  195. me->name, (long long)v, location);
  196. return -EINVAL;
  197. }
  198. }
  199. hi20 = (offset + 0x800) & 0xfffff000;
  200. lo12 = (offset - hi20) & 0xfff;
  201. *location = (*location & 0xfff) | hi20;
  202. *(location + 1) = (*(location + 1) & 0xfffff) | (lo12 << 20);
  203. return 0;
  204. }
  205. static int apply_r_riscv_call_rela(struct module *me, u32 *location,
  206. Elf_Addr v)
  207. {
  208. ptrdiff_t offset = (void *)v - (void *)location;
  209. u32 hi20, lo12;
  210. if (!riscv_insn_valid_32bit_offset(offset)) {
  211. pr_err(
  212. "%s: target %016llx can not be addressed by the 32-bit offset from PC = %p\n",
  213. me->name, (long long)v, location);
  214. return -EINVAL;
  215. }
  216. hi20 = (offset + 0x800) & 0xfffff000;
  217. lo12 = (offset - hi20) & 0xfff;
  218. *location = (*location & 0xfff) | hi20;
  219. *(location + 1) = (*(location + 1) & 0xfffff) | (lo12 << 20);
  220. return 0;
  221. }
  222. static int apply_r_riscv_relax_rela(struct module *me, u32 *location,
  223. Elf_Addr v)
  224. {
  225. return 0;
  226. }
  227. static int apply_r_riscv_align_rela(struct module *me, u32 *location,
  228. Elf_Addr v)
  229. {
  230. pr_err(
  231. "%s: The unexpected relocation type 'R_RISCV_ALIGN' from PC = %p\n",
  232. me->name, location);
  233. return 0; /* Do not return -EINVAL when relocation type is R_RISCV_ALIGN */
  234. }
  235. static int apply_r_riscv_add32_rela(struct module *me, u32 *location,
  236. Elf_Addr v)
  237. {
  238. *(u32 *)location += (u32)v;
  239. return 0;
  240. }
  241. static int apply_r_riscv_add64_rela(struct module *me, u32 *location,
  242. Elf_Addr v)
  243. {
  244. *(u64 *)location += (u64)v;
  245. return 0;
  246. }
  247. static int apply_r_riscv_sub32_rela(struct module *me, u32 *location,
  248. Elf_Addr v)
  249. {
  250. *(u32 *)location -= (u32)v;
  251. return 0;
  252. }
  253. static int apply_r_riscv_sub64_rela(struct module *me, u32 *location,
  254. Elf_Addr v)
  255. {
  256. *(u64 *)location -= (u64)v;
  257. return 0;
  258. }
  259. static int (*reloc_handlers_rela[]) (struct module *me, u32 *location,
  260. Elf_Addr v) = {
  261. [R_RISCV_32] = apply_r_riscv_32_rela,
  262. [R_RISCV_64] = apply_r_riscv_64_rela,
  263. [R_RISCV_BRANCH] = apply_r_riscv_branch_rela,
  264. [R_RISCV_JAL] = apply_r_riscv_jal_rela,
  265. [R_RISCV_RVC_BRANCH] = apply_r_riscv_rcv_branch_rela,
  266. [R_RISCV_RVC_JUMP] = apply_r_riscv_rvc_jump_rela,
  267. [R_RISCV_PCREL_HI20] = apply_r_riscv_pcrel_hi20_rela,
  268. [R_RISCV_PCREL_LO12_I] = apply_r_riscv_pcrel_lo12_i_rela,
  269. [R_RISCV_PCREL_LO12_S] = apply_r_riscv_pcrel_lo12_s_rela,
  270. [R_RISCV_HI20] = apply_r_riscv_hi20_rela,
  271. [R_RISCV_LO12_I] = apply_r_riscv_lo12_i_rela,
  272. [R_RISCV_LO12_S] = apply_r_riscv_lo12_s_rela,
  273. [R_RISCV_GOT_HI20] = apply_r_riscv_got_hi20_rela,
  274. [R_RISCV_CALL_PLT] = apply_r_riscv_call_plt_rela,
  275. [R_RISCV_CALL] = apply_r_riscv_call_rela,
  276. [R_RISCV_RELAX] = apply_r_riscv_relax_rela,
  277. [R_RISCV_ALIGN] = apply_r_riscv_align_rela,
  278. [R_RISCV_ADD32] = apply_r_riscv_add32_rela,
  279. [R_RISCV_ADD64] = apply_r_riscv_add64_rela,
  280. [R_RISCV_SUB32] = apply_r_riscv_sub32_rela,
  281. [R_RISCV_SUB64] = apply_r_riscv_sub64_rela,
  282. };
  283. int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
  284. unsigned int symindex, unsigned int relsec,
  285. struct module *me)
  286. {
  287. Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr;
  288. int (*handler)(struct module *me, u32 *location, Elf_Addr v);
  289. Elf_Sym *sym;
  290. u32 *location;
  291. unsigned int i, type;
  292. Elf_Addr v;
  293. int res;
  294. pr_debug("Applying relocate section %u to %u\n", relsec,
  295. sechdrs[relsec].sh_info);
  296. for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
  297. /* This is where to make the change */
  298. location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
  299. + rel[i].r_offset;
  300. /* This is the symbol it is referring to */
  301. sym = (Elf_Sym *)sechdrs[symindex].sh_addr
  302. + ELF_RISCV_R_SYM(rel[i].r_info);
  303. if (IS_ERR_VALUE(sym->st_value)) {
  304. /* Ignore unresolved weak symbol */
  305. if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
  306. continue;
  307. pr_warn("%s: Unknown symbol %s\n",
  308. me->name, strtab + sym->st_name);
  309. return -ENOENT;
  310. }
  311. type = ELF_RISCV_R_TYPE(rel[i].r_info);
  312. if (type < ARRAY_SIZE(reloc_handlers_rela))
  313. handler = reloc_handlers_rela[type];
  314. else
  315. handler = NULL;
  316. if (!handler) {
  317. pr_err("%s: Unknown relocation type %u\n",
  318. me->name, type);
  319. return -EINVAL;
  320. }
  321. v = sym->st_value + rel[i].r_addend;
  322. if (type == R_RISCV_PCREL_LO12_I || type == R_RISCV_PCREL_LO12_S) {
  323. unsigned int j;
  324. for (j = 0; j < sechdrs[relsec].sh_size / sizeof(*rel); j++) {
  325. unsigned long hi20_loc =
  326. sechdrs[sechdrs[relsec].sh_info].sh_addr
  327. + rel[j].r_offset;
  328. u32 hi20_type = ELF_RISCV_R_TYPE(rel[j].r_info);
  329. /* Find the corresponding HI20 relocation entry */
  330. if (hi20_loc == sym->st_value
  331. && (hi20_type == R_RISCV_PCREL_HI20
  332. || hi20_type == R_RISCV_GOT_HI20)) {
  333. s32 hi20, lo12;
  334. Elf_Sym *hi20_sym =
  335. (Elf_Sym *)sechdrs[symindex].sh_addr
  336. + ELF_RISCV_R_SYM(rel[j].r_info);
  337. unsigned long hi20_sym_val =
  338. hi20_sym->st_value
  339. + rel[j].r_addend;
  340. /* Calculate lo12 */
  341. size_t offset = hi20_sym_val - hi20_loc;
  342. if (IS_ENABLED(CONFIG_MODULE_SECTIONS)
  343. && hi20_type == R_RISCV_GOT_HI20) {
  344. offset = module_emit_got_entry(
  345. me, hi20_sym_val);
  346. offset = offset - hi20_loc;
  347. }
  348. hi20 = (offset + 0x800) & 0xfffff000;
  349. lo12 = offset - hi20;
  350. v = lo12;
  351. break;
  352. }
  353. }
  354. if (j == sechdrs[relsec].sh_size / sizeof(*rel)) {
  355. pr_err(
  356. "%s: Can not find HI20 relocation information\n",
  357. me->name);
  358. return -EINVAL;
  359. }
  360. }
  361. res = handler(me, location, v);
  362. if (res)
  363. return res;
  364. }
  365. return 0;
  366. }
  367. #if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
  368. #define VMALLOC_MODULE_START \
  369. max(PFN_ALIGN((unsigned long)&_end - SZ_2G), VMALLOC_START)
  370. void *module_alloc(unsigned long size)
  371. {
  372. return __vmalloc_node_range(size, 1, VMALLOC_MODULE_START,
  373. VMALLOC_END, GFP_KERNEL,
  374. PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE,
  375. __builtin_return_address(0));
  376. }
  377. #endif