machine_kexec_64.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * handle transition of Linux booting another kernel
  4. * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com>
  5. */
  6. #define pr_fmt(fmt) "kexec: " fmt
  7. #include <linux/mm.h>
  8. #include <linux/kexec.h>
  9. #include <linux/string.h>
  10. #include <linux/gfp.h>
  11. #include <linux/reboot.h>
  12. #include <linux/numa.h>
  13. #include <linux/ftrace.h>
  14. #include <linux/io.h>
  15. #include <linux/suspend.h>
  16. #include <linux/vmalloc.h>
  17. #include <linux/efi.h>
  18. #include <asm/init.h>
  19. #include <asm/tlbflush.h>
  20. #include <asm/mmu_context.h>
  21. #include <asm/io_apic.h>
  22. #include <asm/debugreg.h>
  23. #include <asm/kexec-bzimage64.h>
  24. #include <asm/setup.h>
  25. #include <asm/set_memory.h>
  26. #ifdef CONFIG_ACPI
  27. /*
  28. * Used while adding mapping for ACPI tables.
  29. * Can be reused when other iomem regions need be mapped
  30. */
  31. struct init_pgtable_data {
  32. struct x86_mapping_info *info;
  33. pgd_t *level4p;
  34. };
  35. static int mem_region_callback(struct resource *res, void *arg)
  36. {
  37. struct init_pgtable_data *data = arg;
  38. unsigned long mstart, mend;
  39. mstart = res->start;
  40. mend = mstart + resource_size(res) - 1;
  41. return kernel_ident_mapping_init(data->info, data->level4p, mstart, mend);
  42. }
  43. static int
  44. map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p)
  45. {
  46. struct init_pgtable_data data;
  47. unsigned long flags;
  48. int ret;
  49. data.info = info;
  50. data.level4p = level4p;
  51. flags = IORESOURCE_MEM | IORESOURCE_BUSY;
  52. ret = walk_iomem_res_desc(IORES_DESC_ACPI_TABLES, flags, 0, -1,
  53. &data, mem_region_callback);
  54. if (ret && ret != -EINVAL)
  55. return ret;
  56. /* ACPI tables could be located in ACPI Non-volatile Storage region */
  57. ret = walk_iomem_res_desc(IORES_DESC_ACPI_NV_STORAGE, flags, 0, -1,
  58. &data, mem_region_callback);
  59. if (ret && ret != -EINVAL)
  60. return ret;
  61. return 0;
  62. }
  63. #else
  64. static int map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p) { return 0; }
  65. #endif
  66. #ifdef CONFIG_KEXEC_FILE
  67. const struct kexec_file_ops * const kexec_file_loaders[] = {
  68. &kexec_bzImage64_ops,
  69. NULL
  70. };
  71. #endif
  72. static int
  73. map_efi_systab(struct x86_mapping_info *info, pgd_t *level4p)
  74. {
  75. #ifdef CONFIG_EFI
  76. unsigned long mstart, mend;
  77. if (!efi_enabled(EFI_BOOT))
  78. return 0;
  79. mstart = (boot_params.efi_info.efi_systab |
  80. ((u64)boot_params.efi_info.efi_systab_hi<<32));
  81. if (efi_enabled(EFI_64BIT))
  82. mend = mstart + sizeof(efi_system_table_64_t);
  83. else
  84. mend = mstart + sizeof(efi_system_table_32_t);
  85. if (!mstart)
  86. return 0;
  87. return kernel_ident_mapping_init(info, level4p, mstart, mend);
  88. #endif
  89. return 0;
  90. }
  91. static void free_transition_pgtable(struct kimage *image)
  92. {
  93. free_page((unsigned long)image->arch.p4d);
  94. image->arch.p4d = NULL;
  95. free_page((unsigned long)image->arch.pud);
  96. image->arch.pud = NULL;
  97. free_page((unsigned long)image->arch.pmd);
  98. image->arch.pmd = NULL;
  99. free_page((unsigned long)image->arch.pte);
  100. image->arch.pte = NULL;
  101. }
  102. static int init_transition_pgtable(struct kimage *image, pgd_t *pgd)
  103. {
  104. pgprot_t prot = PAGE_KERNEL_EXEC_NOENC;
  105. unsigned long vaddr, paddr;
  106. int result = -ENOMEM;
  107. p4d_t *p4d;
  108. pud_t *pud;
  109. pmd_t *pmd;
  110. pte_t *pte;
  111. vaddr = (unsigned long)relocate_kernel;
  112. paddr = __pa(page_address(image->control_code_page)+PAGE_SIZE);
  113. pgd += pgd_index(vaddr);
  114. if (!pgd_present(*pgd)) {
  115. p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
  116. if (!p4d)
  117. goto err;
  118. image->arch.p4d = p4d;
  119. set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE));
  120. }
  121. p4d = p4d_offset(pgd, vaddr);
  122. if (!p4d_present(*p4d)) {
  123. pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
  124. if (!pud)
  125. goto err;
  126. image->arch.pud = pud;
  127. set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
  128. }
  129. pud = pud_offset(p4d, vaddr);
  130. if (!pud_present(*pud)) {
  131. pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
  132. if (!pmd)
  133. goto err;
  134. image->arch.pmd = pmd;
  135. set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
  136. }
  137. pmd = pmd_offset(pud, vaddr);
  138. if (!pmd_present(*pmd)) {
  139. pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
  140. if (!pte)
  141. goto err;
  142. image->arch.pte = pte;
  143. set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
  144. }
  145. pte = pte_offset_kernel(pmd, vaddr);
  146. if (sev_active())
  147. prot = PAGE_KERNEL_EXEC;
  148. set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, prot));
  149. return 0;
  150. err:
  151. return result;
  152. }
  153. static void *alloc_pgt_page(void *data)
  154. {
  155. struct kimage *image = (struct kimage *)data;
  156. struct page *page;
  157. void *p = NULL;
  158. page = kimage_alloc_control_pages(image, 0);
  159. if (page) {
  160. p = page_address(page);
  161. clear_page(p);
  162. }
  163. return p;
  164. }
  165. static int init_pgtable(struct kimage *image, unsigned long start_pgtable)
  166. {
  167. struct x86_mapping_info info = {
  168. .alloc_pgt_page = alloc_pgt_page,
  169. .context = image,
  170. .page_flag = __PAGE_KERNEL_LARGE_EXEC,
  171. .kernpg_flag = _KERNPG_TABLE_NOENC,
  172. };
  173. unsigned long mstart, mend;
  174. pgd_t *level4p;
  175. int result;
  176. int i;
  177. level4p = (pgd_t *)__va(start_pgtable);
  178. clear_page(level4p);
  179. if (sev_active()) {
  180. info.page_flag |= _PAGE_ENC;
  181. info.kernpg_flag |= _PAGE_ENC;
  182. }
  183. if (direct_gbpages)
  184. info.direct_gbpages = true;
  185. for (i = 0; i < nr_pfn_mapped; i++) {
  186. mstart = pfn_mapped[i].start << PAGE_SHIFT;
  187. mend = pfn_mapped[i].end << PAGE_SHIFT;
  188. result = kernel_ident_mapping_init(&info,
  189. level4p, mstart, mend);
  190. if (result)
  191. return result;
  192. }
  193. /*
  194. * segments's mem ranges could be outside 0 ~ max_pfn,
  195. * for example when jump back to original kernel from kexeced kernel.
  196. * or first kernel is booted with user mem map, and second kernel
  197. * could be loaded out of that range.
  198. */
  199. for (i = 0; i < image->nr_segments; i++) {
  200. mstart = image->segment[i].mem;
  201. mend = mstart + image->segment[i].memsz;
  202. result = kernel_ident_mapping_init(&info,
  203. level4p, mstart, mend);
  204. if (result)
  205. return result;
  206. }
  207. /*
  208. * Prepare EFI systab and ACPI tables for kexec kernel since they are
  209. * not covered by pfn_mapped.
  210. */
  211. result = map_efi_systab(&info, level4p);
  212. if (result)
  213. return result;
  214. result = map_acpi_tables(&info, level4p);
  215. if (result)
  216. return result;
  217. return init_transition_pgtable(image, level4p);
  218. }
  219. static void set_idt(void *newidt, u16 limit)
  220. {
  221. struct desc_ptr curidt;
  222. /* x86-64 supports unaliged loads & stores */
  223. curidt.size = limit;
  224. curidt.address = (unsigned long)newidt;
  225. __asm__ __volatile__ (
  226. "lidtq %0\n"
  227. : : "m" (curidt)
  228. );
  229. };
  230. static void set_gdt(void *newgdt, u16 limit)
  231. {
  232. struct desc_ptr curgdt;
  233. /* x86-64 supports unaligned loads & stores */
  234. curgdt.size = limit;
  235. curgdt.address = (unsigned long)newgdt;
  236. __asm__ __volatile__ (
  237. "lgdtq %0\n"
  238. : : "m" (curgdt)
  239. );
  240. };
  241. static void load_segments(void)
  242. {
  243. __asm__ __volatile__ (
  244. "\tmovl %0,%%ds\n"
  245. "\tmovl %0,%%es\n"
  246. "\tmovl %0,%%ss\n"
  247. "\tmovl %0,%%fs\n"
  248. "\tmovl %0,%%gs\n"
  249. : : "a" (__KERNEL_DS) : "memory"
  250. );
  251. }
  252. int machine_kexec_prepare(struct kimage *image)
  253. {
  254. unsigned long start_pgtable;
  255. int result;
  256. /* Calculate the offsets */
  257. start_pgtable = page_to_pfn(image->control_code_page) << PAGE_SHIFT;
  258. /* Setup the identity mapped 64bit page table */
  259. result = init_pgtable(image, start_pgtable);
  260. if (result)
  261. return result;
  262. return 0;
  263. }
  264. void machine_kexec_cleanup(struct kimage *image)
  265. {
  266. free_transition_pgtable(image);
  267. }
  268. /*
  269. * Do not allocate memory (or fail in any way) in machine_kexec().
  270. * We are past the point of no return, committed to rebooting now.
  271. */
  272. void machine_kexec(struct kimage *image)
  273. {
  274. unsigned long page_list[PAGES_NR];
  275. void *control_page;
  276. int save_ftrace_enabled;
  277. #ifdef CONFIG_KEXEC_JUMP
  278. if (image->preserve_context)
  279. save_processor_state();
  280. #endif
  281. save_ftrace_enabled = __ftrace_enabled_save();
  282. /* Interrupts aren't acceptable while we reboot */
  283. local_irq_disable();
  284. hw_breakpoint_disable();
  285. if (image->preserve_context) {
  286. #ifdef CONFIG_X86_IO_APIC
  287. /*
  288. * We need to put APICs in legacy mode so that we can
  289. * get timer interrupts in second kernel. kexec/kdump
  290. * paths already have calls to restore_boot_irq_mode()
  291. * in one form or other. kexec jump path also need one.
  292. */
  293. clear_IO_APIC();
  294. restore_boot_irq_mode();
  295. #endif
  296. }
  297. control_page = page_address(image->control_code_page) + PAGE_SIZE;
  298. memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
  299. page_list[PA_CONTROL_PAGE] = virt_to_phys(control_page);
  300. page_list[VA_CONTROL_PAGE] = (unsigned long)control_page;
  301. page_list[PA_TABLE_PAGE] =
  302. (unsigned long)__pa(page_address(image->control_code_page));
  303. if (image->type == KEXEC_TYPE_DEFAULT)
  304. page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page)
  305. << PAGE_SHIFT);
  306. /*
  307. * The segment registers are funny things, they have both a
  308. * visible and an invisible part. Whenever the visible part is
  309. * set to a specific selector, the invisible part is loaded
  310. * with from a table in memory. At no other time is the
  311. * descriptor table in memory accessed.
  312. *
  313. * I take advantage of this here by force loading the
  314. * segments, before I zap the gdt with an invalid value.
  315. */
  316. load_segments();
  317. /*
  318. * The gdt & idt are now invalid.
  319. * If you want to load them you must set up your own idt & gdt.
  320. */
  321. set_gdt(phys_to_virt(0), 0);
  322. set_idt(phys_to_virt(0), 0);
  323. /* now call it */
  324. image->start = relocate_kernel((unsigned long)image->head,
  325. (unsigned long)page_list,
  326. image->start,
  327. image->preserve_context,
  328. sme_active());
  329. #ifdef CONFIG_KEXEC_JUMP
  330. if (image->preserve_context)
  331. restore_processor_state();
  332. #endif
  333. __ftrace_enabled_restore(save_ftrace_enabled);
  334. }
  335. /* arch-dependent functionality related to kexec file-based syscall */
  336. #ifdef CONFIG_KEXEC_FILE
  337. void *arch_kexec_kernel_image_load(struct kimage *image)
  338. {
  339. vfree(image->arch.elf_headers);
  340. image->arch.elf_headers = NULL;
  341. if (!image->fops || !image->fops->load)
  342. return ERR_PTR(-ENOEXEC);
  343. return image->fops->load(image, image->kernel_buf,
  344. image->kernel_buf_len, image->initrd_buf,
  345. image->initrd_buf_len, image->cmdline_buf,
  346. image->cmdline_buf_len);
  347. }
  348. /*
  349. * Apply purgatory relocations.
  350. *
  351. * @pi: Purgatory to be relocated.
  352. * @section: Section relocations applying to.
  353. * @relsec: Section containing RELAs.
  354. * @symtabsec: Corresponding symtab.
  355. *
  356. * TODO: Some of the code belongs to generic code. Move that in kexec.c.
  357. */
  358. int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
  359. Elf_Shdr *section, const Elf_Shdr *relsec,
  360. const Elf_Shdr *symtabsec)
  361. {
  362. unsigned int i;
  363. Elf64_Rela *rel;
  364. Elf64_Sym *sym;
  365. void *location;
  366. unsigned long address, sec_base, value;
  367. const char *strtab, *name, *shstrtab;
  368. const Elf_Shdr *sechdrs;
  369. /* String & section header string table */
  370. sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
  371. strtab = (char *)pi->ehdr + sechdrs[symtabsec->sh_link].sh_offset;
  372. shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
  373. rel = (void *)pi->ehdr + relsec->sh_offset;
  374. pr_debug("Applying relocate section %s to %u\n",
  375. shstrtab + relsec->sh_name, relsec->sh_info);
  376. for (i = 0; i < relsec->sh_size / sizeof(*rel); i++) {
  377. /*
  378. * rel[i].r_offset contains byte offset from beginning
  379. * of section to the storage unit affected.
  380. *
  381. * This is location to update. This is temporary buffer
  382. * where section is currently loaded. This will finally be
  383. * loaded to a different address later, pointed to by
  384. * ->sh_addr. kexec takes care of moving it
  385. * (kexec_load_segment()).
  386. */
  387. location = pi->purgatory_buf;
  388. location += section->sh_offset;
  389. location += rel[i].r_offset;
  390. /* Final address of the location */
  391. address = section->sh_addr + rel[i].r_offset;
  392. /*
  393. * rel[i].r_info contains information about symbol table index
  394. * w.r.t which relocation must be made and type of relocation
  395. * to apply. ELF64_R_SYM() and ELF64_R_TYPE() macros get
  396. * these respectively.
  397. */
  398. sym = (void *)pi->ehdr + symtabsec->sh_offset;
  399. sym += ELF64_R_SYM(rel[i].r_info);
  400. if (sym->st_name)
  401. name = strtab + sym->st_name;
  402. else
  403. name = shstrtab + sechdrs[sym->st_shndx].sh_name;
  404. pr_debug("Symbol: %s info: %02x shndx: %02x value=%llx size: %llx\n",
  405. name, sym->st_info, sym->st_shndx, sym->st_value,
  406. sym->st_size);
  407. if (sym->st_shndx == SHN_UNDEF) {
  408. pr_err("Undefined symbol: %s\n", name);
  409. return -ENOEXEC;
  410. }
  411. if (sym->st_shndx == SHN_COMMON) {
  412. pr_err("symbol '%s' in common section\n", name);
  413. return -ENOEXEC;
  414. }
  415. if (sym->st_shndx == SHN_ABS)
  416. sec_base = 0;
  417. else if (sym->st_shndx >= pi->ehdr->e_shnum) {
  418. pr_err("Invalid section %d for symbol %s\n",
  419. sym->st_shndx, name);
  420. return -ENOEXEC;
  421. } else
  422. sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
  423. value = sym->st_value;
  424. value += sec_base;
  425. value += rel[i].r_addend;
  426. switch (ELF64_R_TYPE(rel[i].r_info)) {
  427. case R_X86_64_NONE:
  428. break;
  429. case R_X86_64_64:
  430. *(u64 *)location = value;
  431. break;
  432. case R_X86_64_32:
  433. *(u32 *)location = value;
  434. if (value != *(u32 *)location)
  435. goto overflow;
  436. break;
  437. case R_X86_64_32S:
  438. *(s32 *)location = value;
  439. if ((s64)value != *(s32 *)location)
  440. goto overflow;
  441. break;
  442. case R_X86_64_PC32:
  443. case R_X86_64_PLT32:
  444. value -= (u64)address;
  445. *(u32 *)location = value;
  446. break;
  447. default:
  448. pr_err("Unknown rela relocation: %llu\n",
  449. ELF64_R_TYPE(rel[i].r_info));
  450. return -ENOEXEC;
  451. }
  452. }
  453. return 0;
  454. overflow:
  455. pr_err("Overflow in relocation type %d value 0x%lx\n",
  456. (int)ELF64_R_TYPE(rel[i].r_info), value);
  457. return -ENOEXEC;
  458. }
  459. #endif /* CONFIG_KEXEC_FILE */
  460. static int
  461. kexec_mark_range(unsigned long start, unsigned long end, bool protect)
  462. {
  463. struct page *page;
  464. unsigned int nr_pages;
  465. /*
  466. * For physical range: [start, end]. We must skip the unassigned
  467. * crashk resource with zero-valued "end" member.
  468. */
  469. if (!end || start > end)
  470. return 0;
  471. page = pfn_to_page(start >> PAGE_SHIFT);
  472. nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
  473. if (protect)
  474. return set_pages_ro(page, nr_pages);
  475. else
  476. return set_pages_rw(page, nr_pages);
  477. }
  478. static void kexec_mark_crashkres(bool protect)
  479. {
  480. unsigned long control;
  481. kexec_mark_range(crashk_low_res.start, crashk_low_res.end, protect);
  482. /* Don't touch the control code page used in crash_kexec().*/
  483. control = PFN_PHYS(page_to_pfn(kexec_crash_image->control_code_page));
  484. /* Control code page is located in the 2nd page. */
  485. kexec_mark_range(crashk_res.start, control + PAGE_SIZE - 1, protect);
  486. control += KEXEC_CONTROL_PAGE_SIZE;
  487. kexec_mark_range(control, crashk_res.end, protect);
  488. }
  489. void arch_kexec_protect_crashkres(void)
  490. {
  491. kexec_mark_crashkres(true);
  492. }
  493. void arch_kexec_unprotect_crashkres(void)
  494. {
  495. kexec_mark_crashkres(false);
  496. }
  497. /*
  498. * During a traditional boot under SME, SME will encrypt the kernel,
  499. * so the SME kexec kernel also needs to be un-encrypted in order to
  500. * replicate a normal SME boot.
  501. *
  502. * During a traditional boot under SEV, the kernel has already been
  503. * loaded encrypted, so the SEV kexec kernel needs to be encrypted in
  504. * order to replicate a normal SEV boot.
  505. */
  506. int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, gfp_t gfp)
  507. {
  508. if (sev_active())
  509. return 0;
  510. /*
  511. * If SME is active we need to be sure that kexec pages are
  512. * not encrypted because when we boot to the new kernel the
  513. * pages won't be accessed encrypted (initially).
  514. */
  515. return set_memory_decrypted((unsigned long)vaddr, pages);
  516. }
  517. void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages)
  518. {
  519. if (sev_active())
  520. return;
  521. /*
  522. * If SME is active we need to reset the pages back to being
  523. * an encrypted mapping before freeing them.
  524. */
  525. set_memory_encrypted((unsigned long)vaddr, pages);
  526. }