machine_kexec.c 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * kexec for arm64
  4. *
  5. * Copyright (C) Linaro.
  6. * Copyright (C) Huawei Futurewei Technologies.
  7. */
  8. #include <linux/interrupt.h>
  9. #include <linux/irq.h>
  10. #include <linux/kernel.h>
  11. #include <linux/kexec.h>
  12. #include <linux/page-flags.h>
  13. #include <linux/smp.h>
  14. #include <asm/cacheflush.h>
  15. #include <asm/cpu_ops.h>
  16. #include <asm/daifflags.h>
  17. #include <asm/memory.h>
  18. #include <asm/mmu.h>
  19. #include <asm/mmu_context.h>
  20. #include <asm/page.h>
  21. #include "cpu-reset.h"
  22. /* Global variables for the arm64_relocate_new_kernel routine. */
  23. extern const unsigned char arm64_relocate_new_kernel[];
  24. extern const unsigned long arm64_relocate_new_kernel_size;
  25. /**
  26. * kexec_image_info - For debugging output.
  27. */
  28. #define kexec_image_info(_i) _kexec_image_info(__func__, __LINE__, _i)
  29. static void _kexec_image_info(const char *func, int line,
  30. const struct kimage *kimage)
  31. {
  32. unsigned long i;
  33. pr_debug("%s:%d:\n", func, line);
  34. pr_debug(" kexec kimage info:\n");
  35. pr_debug(" type: %d\n", kimage->type);
  36. pr_debug(" start: %lx\n", kimage->start);
  37. pr_debug(" head: %lx\n", kimage->head);
  38. pr_debug(" nr_segments: %lu\n", kimage->nr_segments);
  39. for (i = 0; i < kimage->nr_segments; i++) {
  40. pr_debug(" segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n",
  41. i,
  42. kimage->segment[i].mem,
  43. kimage->segment[i].mem + kimage->segment[i].memsz,
  44. kimage->segment[i].memsz,
  45. kimage->segment[i].memsz / PAGE_SIZE);
  46. }
  47. }
  48. void machine_kexec_cleanup(struct kimage *kimage)
  49. {
  50. /* Empty routine needed to avoid build errors. */
  51. }
  52. /**
  53. * machine_kexec_prepare - Prepare for a kexec reboot.
  54. *
  55. * Called from the core kexec code when a kernel image is loaded.
  56. * Forbid loading a kexec kernel if we have no way of hotplugging cpus or cpus
  57. * are stuck in the kernel. This avoids a panic once we hit machine_kexec().
  58. */
  59. int machine_kexec_prepare(struct kimage *kimage)
  60. {
  61. kexec_image_info(kimage);
  62. if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) {
  63. pr_err("Can't kexec: CPUs are stuck in the kernel.\n");
  64. return -EBUSY;
  65. }
  66. return 0;
  67. }
  68. /**
  69. * kexec_list_flush - Helper to flush the kimage list and source pages to PoC.
  70. */
  71. static void kexec_list_flush(struct kimage *kimage)
  72. {
  73. kimage_entry_t *entry;
  74. for (entry = &kimage->head; ; entry++) {
  75. unsigned int flag;
  76. void *addr;
  77. /* flush the list entries. */
  78. __flush_dcache_area(entry, sizeof(kimage_entry_t));
  79. flag = *entry & IND_FLAGS;
  80. if (flag == IND_DONE)
  81. break;
  82. addr = phys_to_virt(*entry & PAGE_MASK);
  83. switch (flag) {
  84. case IND_INDIRECTION:
  85. /* Set entry point just before the new list page. */
  86. entry = (kimage_entry_t *)addr - 1;
  87. break;
  88. case IND_SOURCE:
  89. /* flush the source pages. */
  90. __flush_dcache_area(addr, PAGE_SIZE);
  91. break;
  92. case IND_DESTINATION:
  93. break;
  94. default:
  95. BUG();
  96. }
  97. }
  98. }
  99. /**
  100. * kexec_segment_flush - Helper to flush the kimage segments to PoC.
  101. */
  102. static void kexec_segment_flush(const struct kimage *kimage)
  103. {
  104. unsigned long i;
  105. pr_debug("%s:\n", __func__);
  106. for (i = 0; i < kimage->nr_segments; i++) {
  107. pr_debug(" segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n",
  108. i,
  109. kimage->segment[i].mem,
  110. kimage->segment[i].mem + kimage->segment[i].memsz,
  111. kimage->segment[i].memsz,
  112. kimage->segment[i].memsz / PAGE_SIZE);
  113. __flush_dcache_area(phys_to_virt(kimage->segment[i].mem),
  114. kimage->segment[i].memsz);
  115. }
  116. }
  117. /**
  118. * machine_kexec - Do the kexec reboot.
  119. *
  120. * Called from the core kexec code for a sys_reboot with LINUX_REBOOT_CMD_KEXEC.
  121. */
  122. void machine_kexec(struct kimage *kimage)
  123. {
  124. phys_addr_t reboot_code_buffer_phys;
  125. void *reboot_code_buffer;
  126. bool in_kexec_crash = (kimage == kexec_crash_image);
  127. bool stuck_cpus = cpus_are_stuck_in_kernel();
  128. /*
  129. * New cpus may have become stuck_in_kernel after we loaded the image.
  130. */
  131. BUG_ON(!in_kexec_crash && (stuck_cpus || (num_online_cpus() > 1)));
  132. WARN(in_kexec_crash && (stuck_cpus || smp_crash_stop_failed()),
  133. "Some CPUs may be stale, kdump will be unreliable.\n");
  134. reboot_code_buffer_phys = page_to_phys(kimage->control_code_page);
  135. reboot_code_buffer = phys_to_virt(reboot_code_buffer_phys);
  136. kexec_image_info(kimage);
  137. /*
  138. * Copy arm64_relocate_new_kernel to the reboot_code_buffer for use
  139. * after the kernel is shut down.
  140. */
  141. memcpy(reboot_code_buffer, arm64_relocate_new_kernel,
  142. arm64_relocate_new_kernel_size);
  143. /* Flush the reboot_code_buffer in preparation for its execution. */
  144. __flush_dcache_area(reboot_code_buffer, arm64_relocate_new_kernel_size);
  145. /*
  146. * Although we've killed off the secondary CPUs, we don't update
  147. * the online mask if we're handling a crash kernel and consequently
  148. * need to avoid flush_icache_range(), which will attempt to IPI
  149. * the offline CPUs. Therefore, we must use the __* variant here.
  150. */
  151. __flush_icache_range((uintptr_t)reboot_code_buffer,
  152. (uintptr_t)reboot_code_buffer +
  153. arm64_relocate_new_kernel_size);
  154. /* Flush the kimage list and its buffers. */
  155. kexec_list_flush(kimage);
  156. /* Flush the new image if already in place. */
  157. if ((kimage != kexec_crash_image) && (kimage->head & IND_DONE))
  158. kexec_segment_flush(kimage);
  159. pr_info("Bye!\n");
  160. local_daif_mask();
  161. /*
  162. * cpu_soft_restart will shutdown the MMU, disable data caches, then
  163. * transfer control to the reboot_code_buffer which contains a copy of
  164. * the arm64_relocate_new_kernel routine. arm64_relocate_new_kernel
  165. * uses physical addressing to relocate the new image to its final
  166. * position and transfers control to the image entry point when the
  167. * relocation is complete.
  168. * In kexec case, kimage->start points to purgatory assuming that
  169. * kernel entry and dtb address are embedded in purgatory by
  170. * userspace (kexec-tools).
  171. * In kexec_file case, the kernel starts directly without purgatory.
  172. */
  173. cpu_soft_restart(reboot_code_buffer_phys, kimage->head, kimage->start,
  174. #ifdef CONFIG_KEXEC_FILE
  175. kimage->arch.dtb_mem);
  176. #else
  177. 0);
  178. #endif
  179. BUG(); /* Should never get here. */
  180. }
  181. static void machine_kexec_mask_interrupts(void)
  182. {
  183. unsigned int i;
  184. struct irq_desc *desc;
  185. for_each_irq_desc(i, desc) {
  186. struct irq_chip *chip;
  187. int ret;
  188. chip = irq_desc_get_chip(desc);
  189. if (!chip)
  190. continue;
  191. /*
  192. * First try to remove the active state. If this
  193. * fails, try to EOI the interrupt.
  194. */
  195. ret = irq_set_irqchip_state(i, IRQCHIP_STATE_ACTIVE, false);
  196. if (ret && irqd_irq_inprogress(&desc->irq_data) &&
  197. chip->irq_eoi)
  198. chip->irq_eoi(&desc->irq_data);
  199. if (chip->irq_mask)
  200. chip->irq_mask(&desc->irq_data);
  201. if (chip->irq_disable && !irqd_irq_disabled(&desc->irq_data))
  202. chip->irq_disable(&desc->irq_data);
  203. }
  204. }
  205. /**
  206. * machine_crash_shutdown - shutdown non-crashing cpus and save registers
  207. */
  208. void machine_crash_shutdown(struct pt_regs *regs)
  209. {
  210. local_irq_disable();
  211. /* shutdown non-crashing cpus */
  212. crash_smp_send_stop();
  213. /* for crashing cpu */
  214. crash_save_cpu(regs, smp_processor_id());
  215. machine_kexec_mask_interrupts();
  216. pr_info("Starting crashdump kernel...\n");
  217. }
  218. void arch_kexec_protect_crashkres(void)
  219. {
  220. int i;
  221. kexec_segment_flush(kexec_crash_image);
  222. for (i = 0; i < kexec_crash_image->nr_segments; i++)
  223. set_memory_valid(
  224. __phys_to_virt(kexec_crash_image->segment[i].mem),
  225. kexec_crash_image->segment[i].memsz >> PAGE_SHIFT, 0);
  226. }
  227. void arch_kexec_unprotect_crashkres(void)
  228. {
  229. int i;
  230. for (i = 0; i < kexec_crash_image->nr_segments; i++)
  231. set_memory_valid(
  232. __phys_to_virt(kexec_crash_image->segment[i].mem),
  233. kexec_crash_image->segment[i].memsz >> PAGE_SHIFT, 1);
  234. }
  235. #ifdef CONFIG_HIBERNATION
  236. /*
  237. * To preserve the crash dump kernel image, the relevant memory segments
  238. * should be mapped again around the hibernation.
  239. */
  240. void crash_prepare_suspend(void)
  241. {
  242. if (kexec_crash_image)
  243. arch_kexec_unprotect_crashkres();
  244. }
  245. void crash_post_resume(void)
  246. {
  247. if (kexec_crash_image)
  248. arch_kexec_protect_crashkres();
  249. }
  250. /*
  251. * crash_is_nosave
  252. *
  253. * Return true only if a page is part of reserved memory for crash dump kernel,
  254. * but does not hold any data of loaded kernel image.
  255. *
  256. * Note that all the pages in crash dump kernel memory have been initially
  257. * marked as Reserved as memory was allocated via memblock_reserve().
  258. *
  259. * In hibernation, the pages which are Reserved and yet "nosave" are excluded
  260. * from the hibernation iamge. crash_is_nosave() does thich check for crash
  261. * dump kernel and will reduce the total size of hibernation image.
  262. */
  263. bool crash_is_nosave(unsigned long pfn)
  264. {
  265. int i;
  266. phys_addr_t addr;
  267. if (!crashk_res.end)
  268. return false;
  269. /* in reserved memory? */
  270. addr = __pfn_to_phys(pfn);
  271. if ((addr < crashk_res.start) || (crashk_res.end < addr))
  272. return false;
  273. if (!kexec_crash_image)
  274. return true;
  275. /* not part of loaded kernel image? */
  276. for (i = 0; i < kexec_crash_image->nr_segments; i++)
  277. if (addr >= kexec_crash_image->segment[i].mem &&
  278. addr < (kexec_crash_image->segment[i].mem +
  279. kexec_crash_image->segment[i].memsz))
  280. return false;
  281. return true;
  282. }
  283. void crash_free_reserved_phys_range(unsigned long begin, unsigned long end)
  284. {
  285. unsigned long addr;
  286. struct page *page;
  287. for (addr = begin; addr < end; addr += PAGE_SIZE) {
  288. page = phys_to_page(addr);
  289. free_reserved_page(page);
  290. }
  291. }
  292. #endif /* CONFIG_HIBERNATION */