kexec_file.c 33 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * kexec: kexec_file_load system call
  4. *
  5. * Copyright (C) 2014 Red Hat Inc.
  6. * Authors:
  7. * Vivek Goyal <vgoyal@redhat.com>
  8. */
  9. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  10. #include <linux/capability.h>
  11. #include <linux/mm.h>
  12. #include <linux/file.h>
  13. #include <linux/slab.h>
  14. #include <linux/kexec.h>
  15. #include <linux/memblock.h>
  16. #include <linux/mutex.h>
  17. #include <linux/list.h>
  18. #include <linux/fs.h>
  19. #include <linux/ima.h>
  20. #include <crypto/hash.h>
  21. #include <crypto/sha.h>
  22. #include <linux/elf.h>
  23. #include <linux/elfcore.h>
  24. #include <linux/kernel.h>
  25. #include <linux/kernel_read_file.h>
  26. #include <linux/syscalls.h>
  27. #include <linux/vmalloc.h>
  28. #include "kexec_internal.h"
  29. static int kexec_calculate_store_digests(struct kimage *image);
  30. /*
  31. * Currently this is the only default function that is exported as some
  32. * architectures need it to do additional handlings.
  33. * In the future, other default functions may be exported too if required.
  34. */
  35. int kexec_image_probe_default(struct kimage *image, void *buf,
  36. unsigned long buf_len)
  37. {
  38. const struct kexec_file_ops * const *fops;
  39. int ret = -ENOEXEC;
  40. for (fops = &kexec_file_loaders[0]; *fops && (*fops)->probe; ++fops) {
  41. ret = (*fops)->probe(buf, buf_len);
  42. if (!ret) {
  43. image->fops = *fops;
  44. return ret;
  45. }
  46. }
  47. return ret;
  48. }
  49. /* Architectures can provide this probe function */
  50. int __weak arch_kexec_kernel_image_probe(struct kimage *image, void *buf,
  51. unsigned long buf_len)
  52. {
  53. return kexec_image_probe_default(image, buf, buf_len);
  54. }
  55. static void *kexec_image_load_default(struct kimage *image)
  56. {
  57. if (!image->fops || !image->fops->load)
  58. return ERR_PTR(-ENOEXEC);
  59. return image->fops->load(image, image->kernel_buf,
  60. image->kernel_buf_len, image->initrd_buf,
  61. image->initrd_buf_len, image->cmdline_buf,
  62. image->cmdline_buf_len);
  63. }
  64. void * __weak arch_kexec_kernel_image_load(struct kimage *image)
  65. {
  66. return kexec_image_load_default(image);
  67. }
  68. int kexec_image_post_load_cleanup_default(struct kimage *image)
  69. {
  70. if (!image->fops || !image->fops->cleanup)
  71. return 0;
  72. return image->fops->cleanup(image->image_loader_data);
  73. }
  74. int __weak arch_kimage_file_post_load_cleanup(struct kimage *image)
  75. {
  76. return kexec_image_post_load_cleanup_default(image);
  77. }
  78. #ifdef CONFIG_KEXEC_SIG
  79. static int kexec_image_verify_sig_default(struct kimage *image, void *buf,
  80. unsigned long buf_len)
  81. {
  82. if (!image->fops || !image->fops->verify_sig) {
  83. pr_debug("kernel loader does not support signature verification.\n");
  84. return -EKEYREJECTED;
  85. }
  86. return image->fops->verify_sig(buf, buf_len);
  87. }
  88. int __weak arch_kexec_kernel_verify_sig(struct kimage *image, void *buf,
  89. unsigned long buf_len)
  90. {
  91. return kexec_image_verify_sig_default(image, buf, buf_len);
  92. }
  93. #endif
  94. /*
  95. * arch_kexec_apply_relocations_add - apply relocations of type RELA
  96. * @pi: Purgatory to be relocated.
  97. * @section: Section relocations applying to.
  98. * @relsec: Section containing RELAs.
  99. * @symtab: Corresponding symtab.
  100. *
  101. * Return: 0 on success, negative errno on error.
  102. */
  103. int __weak
  104. arch_kexec_apply_relocations_add(struct purgatory_info *pi, Elf_Shdr *section,
  105. const Elf_Shdr *relsec, const Elf_Shdr *symtab)
  106. {
  107. pr_err("RELA relocation unsupported.\n");
  108. return -ENOEXEC;
  109. }
  110. /*
  111. * arch_kexec_apply_relocations - apply relocations of type REL
  112. * @pi: Purgatory to be relocated.
  113. * @section: Section relocations applying to.
  114. * @relsec: Section containing RELs.
  115. * @symtab: Corresponding symtab.
  116. *
  117. * Return: 0 on success, negative errno on error.
  118. */
  119. int __weak
  120. arch_kexec_apply_relocations(struct purgatory_info *pi, Elf_Shdr *section,
  121. const Elf_Shdr *relsec, const Elf_Shdr *symtab)
  122. {
  123. pr_err("REL relocation unsupported.\n");
  124. return -ENOEXEC;
  125. }
  126. /*
  127. * Free up memory used by kernel, initrd, and command line. This is temporary
  128. * memory allocation which is not needed any more after these buffers have
  129. * been loaded into separate segments and have been copied elsewhere.
  130. */
  131. void kimage_file_post_load_cleanup(struct kimage *image)
  132. {
  133. struct purgatory_info *pi = &image->purgatory_info;
  134. vfree(image->kernel_buf);
  135. image->kernel_buf = NULL;
  136. vfree(image->initrd_buf);
  137. image->initrd_buf = NULL;
  138. kfree(image->cmdline_buf);
  139. image->cmdline_buf = NULL;
  140. vfree(pi->purgatory_buf);
  141. pi->purgatory_buf = NULL;
  142. vfree(pi->sechdrs);
  143. pi->sechdrs = NULL;
  144. #ifdef CONFIG_IMA_KEXEC
  145. vfree(image->ima_buffer);
  146. image->ima_buffer = NULL;
  147. #endif /* CONFIG_IMA_KEXEC */
  148. /* See if architecture has anything to cleanup post load */
  149. arch_kimage_file_post_load_cleanup(image);
  150. /*
  151. * Above call should have called into bootloader to free up
  152. * any data stored in kimage->image_loader_data. It should
  153. * be ok now to free it up.
  154. */
  155. kfree(image->image_loader_data);
  156. image->image_loader_data = NULL;
  157. }
  158. #ifdef CONFIG_KEXEC_SIG
  159. static int
  160. kimage_validate_signature(struct kimage *image)
  161. {
  162. int ret;
  163. ret = arch_kexec_kernel_verify_sig(image, image->kernel_buf,
  164. image->kernel_buf_len);
  165. if (ret) {
  166. if (IS_ENABLED(CONFIG_KEXEC_SIG_FORCE)) {
  167. pr_notice("Enforced kernel signature verification failed (%d).\n", ret);
  168. return ret;
  169. }
  170. /*
  171. * If IMA is guaranteed to appraise a signature on the kexec
  172. * image, permit it even if the kernel is otherwise locked
  173. * down.
  174. */
  175. if (!ima_appraise_signature(READING_KEXEC_IMAGE) &&
  176. security_locked_down(LOCKDOWN_KEXEC))
  177. return -EPERM;
  178. pr_debug("kernel signature verification failed (%d).\n", ret);
  179. }
  180. return 0;
  181. }
  182. #endif
  183. /*
  184. * In file mode list of segments is prepared by kernel. Copy relevant
  185. * data from user space, do error checking, prepare segment list
  186. */
  187. static int
  188. kimage_file_prepare_segments(struct kimage *image, int kernel_fd, int initrd_fd,
  189. const char __user *cmdline_ptr,
  190. unsigned long cmdline_len, unsigned flags)
  191. {
  192. int ret;
  193. void *ldata;
  194. ret = kernel_read_file_from_fd(kernel_fd, 0, &image->kernel_buf,
  195. INT_MAX, NULL, READING_KEXEC_IMAGE);
  196. if (ret < 0)
  197. return ret;
  198. image->kernel_buf_len = ret;
  199. /* Call arch image probe handlers */
  200. ret = arch_kexec_kernel_image_probe(image, image->kernel_buf,
  201. image->kernel_buf_len);
  202. if (ret)
  203. goto out;
  204. #ifdef CONFIG_KEXEC_SIG
  205. ret = kimage_validate_signature(image);
  206. if (ret)
  207. goto out;
  208. #endif
  209. /* It is possible that there no initramfs is being loaded */
  210. if (!(flags & KEXEC_FILE_NO_INITRAMFS)) {
  211. ret = kernel_read_file_from_fd(initrd_fd, 0, &image->initrd_buf,
  212. INT_MAX, NULL,
  213. READING_KEXEC_INITRAMFS);
  214. if (ret < 0)
  215. goto out;
  216. image->initrd_buf_len = ret;
  217. ret = 0;
  218. }
  219. if (cmdline_len) {
  220. image->cmdline_buf = memdup_user(cmdline_ptr, cmdline_len);
  221. if (IS_ERR(image->cmdline_buf)) {
  222. ret = PTR_ERR(image->cmdline_buf);
  223. image->cmdline_buf = NULL;
  224. goto out;
  225. }
  226. image->cmdline_buf_len = cmdline_len;
  227. /* command line should be a string with last byte null */
  228. if (image->cmdline_buf[cmdline_len - 1] != '\0') {
  229. ret = -EINVAL;
  230. goto out;
  231. }
  232. ima_kexec_cmdline(kernel_fd, image->cmdline_buf,
  233. image->cmdline_buf_len - 1);
  234. }
  235. /* IMA needs to pass the measurement list to the next kernel. */
  236. ima_add_kexec_buffer(image);
  237. /* Call arch image load handlers */
  238. ldata = arch_kexec_kernel_image_load(image);
  239. if (IS_ERR(ldata)) {
  240. ret = PTR_ERR(ldata);
  241. goto out;
  242. }
  243. image->image_loader_data = ldata;
  244. out:
  245. /* In case of error, free up all allocated memory in this function */
  246. if (ret)
  247. kimage_file_post_load_cleanup(image);
  248. return ret;
  249. }
  250. static int
  251. kimage_file_alloc_init(struct kimage **rimage, int kernel_fd,
  252. int initrd_fd, const char __user *cmdline_ptr,
  253. unsigned long cmdline_len, unsigned long flags)
  254. {
  255. int ret;
  256. struct kimage *image;
  257. bool kexec_on_panic = flags & KEXEC_FILE_ON_CRASH;
  258. image = do_kimage_alloc_init();
  259. if (!image)
  260. return -ENOMEM;
  261. image->file_mode = 1;
  262. if (kexec_on_panic) {
  263. /* Enable special crash kernel control page alloc policy. */
  264. image->control_page = crashk_res.start;
  265. image->type = KEXEC_TYPE_CRASH;
  266. }
  267. ret = kimage_file_prepare_segments(image, kernel_fd, initrd_fd,
  268. cmdline_ptr, cmdline_len, flags);
  269. if (ret)
  270. goto out_free_image;
  271. ret = sanity_check_segment_list(image);
  272. if (ret)
  273. goto out_free_post_load_bufs;
  274. ret = -ENOMEM;
  275. image->control_code_page = kimage_alloc_control_pages(image,
  276. get_order(KEXEC_CONTROL_PAGE_SIZE));
  277. if (!image->control_code_page) {
  278. pr_err("Could not allocate control_code_buffer\n");
  279. goto out_free_post_load_bufs;
  280. }
  281. if (!kexec_on_panic) {
  282. image->swap_page = kimage_alloc_control_pages(image, 0);
  283. if (!image->swap_page) {
  284. pr_err("Could not allocate swap buffer\n");
  285. goto out_free_control_pages;
  286. }
  287. }
  288. *rimage = image;
  289. return 0;
  290. out_free_control_pages:
  291. kimage_free_page_list(&image->control_pages);
  292. out_free_post_load_bufs:
  293. kimage_file_post_load_cleanup(image);
  294. out_free_image:
  295. kfree(image);
  296. return ret;
  297. }
  298. SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd,
  299. unsigned long, cmdline_len, const char __user *, cmdline_ptr,
  300. unsigned long, flags)
  301. {
  302. int ret = 0, i;
  303. struct kimage **dest_image, *image;
  304. /* We only trust the superuser with rebooting the system. */
  305. if (!capable(CAP_SYS_BOOT) || kexec_load_disabled)
  306. return -EPERM;
  307. /* Make sure we have a legal set of flags */
  308. if (flags != (flags & KEXEC_FILE_FLAGS))
  309. return -EINVAL;
  310. image = NULL;
  311. if (!mutex_trylock(&kexec_mutex))
  312. return -EBUSY;
  313. dest_image = &kexec_image;
  314. if (flags & KEXEC_FILE_ON_CRASH) {
  315. dest_image = &kexec_crash_image;
  316. if (kexec_crash_image)
  317. arch_kexec_unprotect_crashkres();
  318. }
  319. if (flags & KEXEC_FILE_UNLOAD)
  320. goto exchange;
  321. /*
  322. * In case of crash, new kernel gets loaded in reserved region. It is
  323. * same memory where old crash kernel might be loaded. Free any
  324. * current crash dump kernel before we corrupt it.
  325. */
  326. if (flags & KEXEC_FILE_ON_CRASH)
  327. kimage_free(xchg(&kexec_crash_image, NULL));
  328. ret = kimage_file_alloc_init(&image, kernel_fd, initrd_fd, cmdline_ptr,
  329. cmdline_len, flags);
  330. if (ret)
  331. goto out;
  332. ret = machine_kexec_prepare(image);
  333. if (ret)
  334. goto out;
  335. /*
  336. * Some architecture(like S390) may touch the crash memory before
  337. * machine_kexec_prepare(), we must copy vmcoreinfo data after it.
  338. */
  339. ret = kimage_crash_copy_vmcoreinfo(image);
  340. if (ret)
  341. goto out;
  342. ret = kexec_calculate_store_digests(image);
  343. if (ret)
  344. goto out;
  345. for (i = 0; i < image->nr_segments; i++) {
  346. struct kexec_segment *ksegment;
  347. ksegment = &image->segment[i];
  348. pr_debug("Loading segment %d: buf=0x%p bufsz=0x%zx mem=0x%lx memsz=0x%zx\n",
  349. i, ksegment->buf, ksegment->bufsz, ksegment->mem,
  350. ksegment->memsz);
  351. ret = kimage_load_segment(image, &image->segment[i]);
  352. if (ret)
  353. goto out;
  354. }
  355. kimage_terminate(image);
  356. ret = machine_kexec_post_load(image);
  357. if (ret)
  358. goto out;
  359. /*
  360. * Free up any temporary buffers allocated which are not needed
  361. * after image has been loaded
  362. */
  363. kimage_file_post_load_cleanup(image);
  364. exchange:
  365. image = xchg(dest_image, image);
  366. out:
  367. if ((flags & KEXEC_FILE_ON_CRASH) && kexec_crash_image)
  368. arch_kexec_protect_crashkres();
  369. mutex_unlock(&kexec_mutex);
  370. kimage_free(image);
  371. return ret;
  372. }
  373. static int locate_mem_hole_top_down(unsigned long start, unsigned long end,
  374. struct kexec_buf *kbuf)
  375. {
  376. struct kimage *image = kbuf->image;
  377. unsigned long temp_start, temp_end;
  378. temp_end = min(end, kbuf->buf_max);
  379. temp_start = temp_end - kbuf->memsz;
  380. do {
  381. /* align down start */
  382. temp_start = temp_start & (~(kbuf->buf_align - 1));
  383. if (temp_start < start || temp_start < kbuf->buf_min)
  384. return 0;
  385. temp_end = temp_start + kbuf->memsz - 1;
  386. /*
  387. * Make sure this does not conflict with any of existing
  388. * segments
  389. */
  390. if (kimage_is_destination_range(image, temp_start, temp_end)) {
  391. temp_start = temp_start - PAGE_SIZE;
  392. continue;
  393. }
  394. /* We found a suitable memory range */
  395. break;
  396. } while (1);
  397. /* If we are here, we found a suitable memory range */
  398. kbuf->mem = temp_start;
  399. /* Success, stop navigating through remaining System RAM ranges */
  400. return 1;
  401. }
  402. static int locate_mem_hole_bottom_up(unsigned long start, unsigned long end,
  403. struct kexec_buf *kbuf)
  404. {
  405. struct kimage *image = kbuf->image;
  406. unsigned long temp_start, temp_end;
  407. temp_start = max(start, kbuf->buf_min);
  408. do {
  409. temp_start = ALIGN(temp_start, kbuf->buf_align);
  410. temp_end = temp_start + kbuf->memsz - 1;
  411. if (temp_end > end || temp_end > kbuf->buf_max)
  412. return 0;
  413. /*
  414. * Make sure this does not conflict with any of existing
  415. * segments
  416. */
  417. if (kimage_is_destination_range(image, temp_start, temp_end)) {
  418. temp_start = temp_start + PAGE_SIZE;
  419. continue;
  420. }
  421. /* We found a suitable memory range */
  422. break;
  423. } while (1);
  424. /* If we are here, we found a suitable memory range */
  425. kbuf->mem = temp_start;
  426. /* Success, stop navigating through remaining System RAM ranges */
  427. return 1;
  428. }
  429. static int locate_mem_hole_callback(struct resource *res, void *arg)
  430. {
  431. struct kexec_buf *kbuf = (struct kexec_buf *)arg;
  432. u64 start = res->start, end = res->end;
  433. unsigned long sz = end - start + 1;
  434. /* Returning 0 will take to next memory range */
  435. /* Don't use memory that will be detected and handled by a driver. */
  436. if (res->flags & IORESOURCE_SYSRAM_DRIVER_MANAGED)
  437. return 0;
  438. if (sz < kbuf->memsz)
  439. return 0;
  440. if (end < kbuf->buf_min || start > kbuf->buf_max)
  441. return 0;
  442. /*
  443. * Allocate memory top down with-in ram range. Otherwise bottom up
  444. * allocation.
  445. */
  446. if (kbuf->top_down)
  447. return locate_mem_hole_top_down(start, end, kbuf);
  448. return locate_mem_hole_bottom_up(start, end, kbuf);
  449. }
  450. #ifdef CONFIG_ARCH_KEEP_MEMBLOCK
  451. static int kexec_walk_memblock(struct kexec_buf *kbuf,
  452. int (*func)(struct resource *, void *))
  453. {
  454. int ret = 0;
  455. u64 i;
  456. phys_addr_t mstart, mend;
  457. struct resource res = { };
  458. if (kbuf->image->type == KEXEC_TYPE_CRASH)
  459. return func(&crashk_res, kbuf);
  460. if (kbuf->top_down) {
  461. for_each_free_mem_range_reverse(i, NUMA_NO_NODE, MEMBLOCK_NONE,
  462. &mstart, &mend, NULL) {
  463. /*
  464. * In memblock, end points to the first byte after the
  465. * range while in kexec, end points to the last byte
  466. * in the range.
  467. */
  468. res.start = mstart;
  469. res.end = mend - 1;
  470. ret = func(&res, kbuf);
  471. if (ret)
  472. break;
  473. }
  474. } else {
  475. for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE,
  476. &mstart, &mend, NULL) {
  477. /*
  478. * In memblock, end points to the first byte after the
  479. * range while in kexec, end points to the last byte
  480. * in the range.
  481. */
  482. res.start = mstart;
  483. res.end = mend - 1;
  484. ret = func(&res, kbuf);
  485. if (ret)
  486. break;
  487. }
  488. }
  489. return ret;
  490. }
  491. #else
  492. static int kexec_walk_memblock(struct kexec_buf *kbuf,
  493. int (*func)(struct resource *, void *))
  494. {
  495. return 0;
  496. }
  497. #endif
  498. /**
  499. * kexec_walk_resources - call func(data) on free memory regions
  500. * @kbuf: Context info for the search. Also passed to @func.
  501. * @func: Function to call for each memory region.
  502. *
  503. * Return: The memory walk will stop when func returns a non-zero value
  504. * and that value will be returned. If all free regions are visited without
  505. * func returning non-zero, then zero will be returned.
  506. */
  507. static int kexec_walk_resources(struct kexec_buf *kbuf,
  508. int (*func)(struct resource *, void *))
  509. {
  510. if (kbuf->image->type == KEXEC_TYPE_CRASH)
  511. return walk_iomem_res_desc(crashk_res.desc,
  512. IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
  513. crashk_res.start, crashk_res.end,
  514. kbuf, func);
  515. else
  516. return walk_system_ram_res(0, ULONG_MAX, kbuf, func);
  517. }
  518. /**
  519. * kexec_locate_mem_hole - find free memory for the purgatory or the next kernel
  520. * @kbuf: Parameters for the memory search.
  521. *
  522. * On success, kbuf->mem will have the start address of the memory region found.
  523. *
  524. * Return: 0 on success, negative errno on error.
  525. */
  526. int kexec_locate_mem_hole(struct kexec_buf *kbuf)
  527. {
  528. int ret;
  529. /* Arch knows where to place */
  530. if (kbuf->mem != KEXEC_BUF_MEM_UNKNOWN)
  531. return 0;
  532. if (!IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK))
  533. ret = kexec_walk_resources(kbuf, locate_mem_hole_callback);
  534. else
  535. ret = kexec_walk_memblock(kbuf, locate_mem_hole_callback);
  536. return ret == 1 ? 0 : -EADDRNOTAVAIL;
  537. }
  538. /**
  539. * arch_kexec_locate_mem_hole - Find free memory to place the segments.
  540. * @kbuf: Parameters for the memory search.
  541. *
  542. * On success, kbuf->mem will have the start address of the memory region found.
  543. *
  544. * Return: 0 on success, negative errno on error.
  545. */
  546. int __weak arch_kexec_locate_mem_hole(struct kexec_buf *kbuf)
  547. {
  548. return kexec_locate_mem_hole(kbuf);
  549. }
  550. /**
  551. * kexec_add_buffer - place a buffer in a kexec segment
  552. * @kbuf: Buffer contents and memory parameters.
  553. *
  554. * This function assumes that kexec_mutex is held.
  555. * On successful return, @kbuf->mem will have the physical address of
  556. * the buffer in memory.
  557. *
  558. * Return: 0 on success, negative errno on error.
  559. */
  560. int kexec_add_buffer(struct kexec_buf *kbuf)
  561. {
  562. struct kexec_segment *ksegment;
  563. int ret;
  564. /* Currently adding segment this way is allowed only in file mode */
  565. if (!kbuf->image->file_mode)
  566. return -EINVAL;
  567. if (kbuf->image->nr_segments >= KEXEC_SEGMENT_MAX)
  568. return -EINVAL;
  569. /*
  570. * Make sure we are not trying to add buffer after allocating
  571. * control pages. All segments need to be placed first before
  572. * any control pages are allocated. As control page allocation
  573. * logic goes through list of segments to make sure there are
  574. * no destination overlaps.
  575. */
  576. if (!list_empty(&kbuf->image->control_pages)) {
  577. WARN_ON(1);
  578. return -EINVAL;
  579. }
  580. /* Ensure minimum alignment needed for segments. */
  581. kbuf->memsz = ALIGN(kbuf->memsz, PAGE_SIZE);
  582. kbuf->buf_align = max(kbuf->buf_align, PAGE_SIZE);
  583. /* Walk the RAM ranges and allocate a suitable range for the buffer */
  584. ret = arch_kexec_locate_mem_hole(kbuf);
  585. if (ret)
  586. return ret;
  587. /* Found a suitable memory range */
  588. ksegment = &kbuf->image->segment[kbuf->image->nr_segments];
  589. ksegment->kbuf = kbuf->buffer;
  590. ksegment->bufsz = kbuf->bufsz;
  591. ksegment->mem = kbuf->mem;
  592. ksegment->memsz = kbuf->memsz;
  593. kbuf->image->nr_segments++;
  594. return 0;
  595. }
  596. /* Calculate and store the digest of segments */
  597. static int kexec_calculate_store_digests(struct kimage *image)
  598. {
  599. struct crypto_shash *tfm;
  600. struct shash_desc *desc;
  601. int ret = 0, i, j, zero_buf_sz, sha_region_sz;
  602. size_t desc_size, nullsz;
  603. char *digest;
  604. void *zero_buf;
  605. struct kexec_sha_region *sha_regions;
  606. struct purgatory_info *pi = &image->purgatory_info;
  607. if (!IS_ENABLED(CONFIG_ARCH_HAS_KEXEC_PURGATORY))
  608. return 0;
  609. zero_buf = __va(page_to_pfn(ZERO_PAGE(0)) << PAGE_SHIFT);
  610. zero_buf_sz = PAGE_SIZE;
  611. tfm = crypto_alloc_shash("sha256", 0, 0);
  612. if (IS_ERR(tfm)) {
  613. ret = PTR_ERR(tfm);
  614. goto out;
  615. }
  616. desc_size = crypto_shash_descsize(tfm) + sizeof(*desc);
  617. desc = kzalloc(desc_size, GFP_KERNEL);
  618. if (!desc) {
  619. ret = -ENOMEM;
  620. goto out_free_tfm;
  621. }
  622. sha_region_sz = KEXEC_SEGMENT_MAX * sizeof(struct kexec_sha_region);
  623. sha_regions = vzalloc(sha_region_sz);
  624. if (!sha_regions) {
  625. ret = -ENOMEM;
  626. goto out_free_desc;
  627. }
  628. desc->tfm = tfm;
  629. ret = crypto_shash_init(desc);
  630. if (ret < 0)
  631. goto out_free_sha_regions;
  632. digest = kzalloc(SHA256_DIGEST_SIZE, GFP_KERNEL);
  633. if (!digest) {
  634. ret = -ENOMEM;
  635. goto out_free_sha_regions;
  636. }
  637. for (j = i = 0; i < image->nr_segments; i++) {
  638. struct kexec_segment *ksegment;
  639. ksegment = &image->segment[i];
  640. /*
  641. * Skip purgatory as it will be modified once we put digest
  642. * info in purgatory.
  643. */
  644. if (ksegment->kbuf == pi->purgatory_buf)
  645. continue;
  646. ret = crypto_shash_update(desc, ksegment->kbuf,
  647. ksegment->bufsz);
  648. if (ret)
  649. break;
  650. /*
  651. * Assume rest of the buffer is filled with zero and
  652. * update digest accordingly.
  653. */
  654. nullsz = ksegment->memsz - ksegment->bufsz;
  655. while (nullsz) {
  656. unsigned long bytes = nullsz;
  657. if (bytes > zero_buf_sz)
  658. bytes = zero_buf_sz;
  659. ret = crypto_shash_update(desc, zero_buf, bytes);
  660. if (ret)
  661. break;
  662. nullsz -= bytes;
  663. }
  664. if (ret)
  665. break;
  666. sha_regions[j].start = ksegment->mem;
  667. sha_regions[j].len = ksegment->memsz;
  668. j++;
  669. }
  670. if (!ret) {
  671. ret = crypto_shash_final(desc, digest);
  672. if (ret)
  673. goto out_free_digest;
  674. ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha_regions",
  675. sha_regions, sha_region_sz, 0);
  676. if (ret)
  677. goto out_free_digest;
  678. ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha256_digest",
  679. digest, SHA256_DIGEST_SIZE, 0);
  680. if (ret)
  681. goto out_free_digest;
  682. }
  683. out_free_digest:
  684. kfree(digest);
  685. out_free_sha_regions:
  686. vfree(sha_regions);
  687. out_free_desc:
  688. kfree(desc);
  689. out_free_tfm:
  690. kfree(tfm);
  691. out:
  692. return ret;
  693. }
  694. #ifdef CONFIG_ARCH_HAS_KEXEC_PURGATORY
  695. /*
  696. * kexec_purgatory_setup_kbuf - prepare buffer to load purgatory.
  697. * @pi: Purgatory to be loaded.
  698. * @kbuf: Buffer to setup.
  699. *
  700. * Allocates the memory needed for the buffer. Caller is responsible to free
  701. * the memory after use.
  702. *
  703. * Return: 0 on success, negative errno on error.
  704. */
  705. static int kexec_purgatory_setup_kbuf(struct purgatory_info *pi,
  706. struct kexec_buf *kbuf)
  707. {
  708. const Elf_Shdr *sechdrs;
  709. unsigned long bss_align;
  710. unsigned long bss_sz;
  711. unsigned long align;
  712. int i, ret;
  713. sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
  714. kbuf->buf_align = bss_align = 1;
  715. kbuf->bufsz = bss_sz = 0;
  716. for (i = 0; i < pi->ehdr->e_shnum; i++) {
  717. if (!(sechdrs[i].sh_flags & SHF_ALLOC))
  718. continue;
  719. align = sechdrs[i].sh_addralign;
  720. if (sechdrs[i].sh_type != SHT_NOBITS) {
  721. if (kbuf->buf_align < align)
  722. kbuf->buf_align = align;
  723. kbuf->bufsz = ALIGN(kbuf->bufsz, align);
  724. kbuf->bufsz += sechdrs[i].sh_size;
  725. } else {
  726. if (bss_align < align)
  727. bss_align = align;
  728. bss_sz = ALIGN(bss_sz, align);
  729. bss_sz += sechdrs[i].sh_size;
  730. }
  731. }
  732. kbuf->bufsz = ALIGN(kbuf->bufsz, bss_align);
  733. kbuf->memsz = kbuf->bufsz + bss_sz;
  734. if (kbuf->buf_align < bss_align)
  735. kbuf->buf_align = bss_align;
  736. kbuf->buffer = vzalloc(kbuf->bufsz);
  737. if (!kbuf->buffer)
  738. return -ENOMEM;
  739. pi->purgatory_buf = kbuf->buffer;
  740. ret = kexec_add_buffer(kbuf);
  741. if (ret)
  742. goto out;
  743. return 0;
  744. out:
  745. vfree(pi->purgatory_buf);
  746. pi->purgatory_buf = NULL;
  747. return ret;
  748. }
  749. /*
  750. * kexec_purgatory_setup_sechdrs - prepares the pi->sechdrs buffer.
  751. * @pi: Purgatory to be loaded.
  752. * @kbuf: Buffer prepared to store purgatory.
  753. *
  754. * Allocates the memory needed for the buffer. Caller is responsible to free
  755. * the memory after use.
  756. *
  757. * Return: 0 on success, negative errno on error.
  758. */
  759. static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi,
  760. struct kexec_buf *kbuf)
  761. {
  762. unsigned long bss_addr;
  763. unsigned long offset;
  764. Elf_Shdr *sechdrs;
  765. int i;
  766. /*
  767. * The section headers in kexec_purgatory are read-only. In order to
  768. * have them modifiable make a temporary copy.
  769. */
  770. sechdrs = vzalloc(array_size(sizeof(Elf_Shdr), pi->ehdr->e_shnum));
  771. if (!sechdrs)
  772. return -ENOMEM;
  773. memcpy(sechdrs, (void *)pi->ehdr + pi->ehdr->e_shoff,
  774. pi->ehdr->e_shnum * sizeof(Elf_Shdr));
  775. pi->sechdrs = sechdrs;
  776. offset = 0;
  777. bss_addr = kbuf->mem + kbuf->bufsz;
  778. kbuf->image->start = pi->ehdr->e_entry;
  779. for (i = 0; i < pi->ehdr->e_shnum; i++) {
  780. unsigned long align;
  781. void *src, *dst;
  782. if (!(sechdrs[i].sh_flags & SHF_ALLOC))
  783. continue;
  784. align = sechdrs[i].sh_addralign;
  785. if (sechdrs[i].sh_type == SHT_NOBITS) {
  786. bss_addr = ALIGN(bss_addr, align);
  787. sechdrs[i].sh_addr = bss_addr;
  788. bss_addr += sechdrs[i].sh_size;
  789. continue;
  790. }
  791. offset = ALIGN(offset, align);
  792. if (sechdrs[i].sh_flags & SHF_EXECINSTR &&
  793. pi->ehdr->e_entry >= sechdrs[i].sh_addr &&
  794. pi->ehdr->e_entry < (sechdrs[i].sh_addr
  795. + sechdrs[i].sh_size)) {
  796. kbuf->image->start -= sechdrs[i].sh_addr;
  797. kbuf->image->start += kbuf->mem + offset;
  798. }
  799. src = (void *)pi->ehdr + sechdrs[i].sh_offset;
  800. dst = pi->purgatory_buf + offset;
  801. memcpy(dst, src, sechdrs[i].sh_size);
  802. sechdrs[i].sh_addr = kbuf->mem + offset;
  803. sechdrs[i].sh_offset = offset;
  804. offset += sechdrs[i].sh_size;
  805. }
  806. return 0;
  807. }
  808. static int kexec_apply_relocations(struct kimage *image)
  809. {
  810. int i, ret;
  811. struct purgatory_info *pi = &image->purgatory_info;
  812. const Elf_Shdr *sechdrs;
  813. sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
  814. for (i = 0; i < pi->ehdr->e_shnum; i++) {
  815. const Elf_Shdr *relsec;
  816. const Elf_Shdr *symtab;
  817. Elf_Shdr *section;
  818. relsec = sechdrs + i;
  819. if (relsec->sh_type != SHT_RELA &&
  820. relsec->sh_type != SHT_REL)
  821. continue;
  822. /*
  823. * For section of type SHT_RELA/SHT_REL,
  824. * ->sh_link contains section header index of associated
  825. * symbol table. And ->sh_info contains section header
  826. * index of section to which relocations apply.
  827. */
  828. if (relsec->sh_info >= pi->ehdr->e_shnum ||
  829. relsec->sh_link >= pi->ehdr->e_shnum)
  830. return -ENOEXEC;
  831. section = pi->sechdrs + relsec->sh_info;
  832. symtab = sechdrs + relsec->sh_link;
  833. if (!(section->sh_flags & SHF_ALLOC))
  834. continue;
  835. /*
  836. * symtab->sh_link contain section header index of associated
  837. * string table.
  838. */
  839. if (symtab->sh_link >= pi->ehdr->e_shnum)
  840. /* Invalid section number? */
  841. continue;
  842. /*
  843. * Respective architecture needs to provide support for applying
  844. * relocations of type SHT_RELA/SHT_REL.
  845. */
  846. if (relsec->sh_type == SHT_RELA)
  847. ret = arch_kexec_apply_relocations_add(pi, section,
  848. relsec, symtab);
  849. else if (relsec->sh_type == SHT_REL)
  850. ret = arch_kexec_apply_relocations(pi, section,
  851. relsec, symtab);
  852. if (ret)
  853. return ret;
  854. }
  855. return 0;
  856. }
  857. /*
  858. * kexec_load_purgatory - Load and relocate the purgatory object.
  859. * @image: Image to add the purgatory to.
  860. * @kbuf: Memory parameters to use.
  861. *
  862. * Allocates the memory needed for image->purgatory_info.sechdrs and
  863. * image->purgatory_info.purgatory_buf/kbuf->buffer. Caller is responsible
  864. * to free the memory after use.
  865. *
  866. * Return: 0 on success, negative errno on error.
  867. */
  868. int kexec_load_purgatory(struct kimage *image, struct kexec_buf *kbuf)
  869. {
  870. struct purgatory_info *pi = &image->purgatory_info;
  871. int ret;
  872. if (kexec_purgatory_size <= 0)
  873. return -EINVAL;
  874. pi->ehdr = (const Elf_Ehdr *)kexec_purgatory;
  875. ret = kexec_purgatory_setup_kbuf(pi, kbuf);
  876. if (ret)
  877. return ret;
  878. ret = kexec_purgatory_setup_sechdrs(pi, kbuf);
  879. if (ret)
  880. goto out_free_kbuf;
  881. ret = kexec_apply_relocations(image);
  882. if (ret)
  883. goto out;
  884. return 0;
  885. out:
  886. vfree(pi->sechdrs);
  887. pi->sechdrs = NULL;
  888. out_free_kbuf:
  889. vfree(pi->purgatory_buf);
  890. pi->purgatory_buf = NULL;
  891. return ret;
  892. }
  893. /*
  894. * kexec_purgatory_find_symbol - find a symbol in the purgatory
  895. * @pi: Purgatory to search in.
  896. * @name: Name of the symbol.
  897. *
  898. * Return: pointer to symbol in read-only symtab on success, NULL on error.
  899. */
  900. static const Elf_Sym *kexec_purgatory_find_symbol(struct purgatory_info *pi,
  901. const char *name)
  902. {
  903. const Elf_Shdr *sechdrs;
  904. const Elf_Ehdr *ehdr;
  905. const Elf_Sym *syms;
  906. const char *strtab;
  907. int i, k;
  908. if (!pi->ehdr)
  909. return NULL;
  910. ehdr = pi->ehdr;
  911. sechdrs = (void *)ehdr + ehdr->e_shoff;
  912. for (i = 0; i < ehdr->e_shnum; i++) {
  913. if (sechdrs[i].sh_type != SHT_SYMTAB)
  914. continue;
  915. if (sechdrs[i].sh_link >= ehdr->e_shnum)
  916. /* Invalid strtab section number */
  917. continue;
  918. strtab = (void *)ehdr + sechdrs[sechdrs[i].sh_link].sh_offset;
  919. syms = (void *)ehdr + sechdrs[i].sh_offset;
  920. /* Go through symbols for a match */
  921. for (k = 0; k < sechdrs[i].sh_size/sizeof(Elf_Sym); k++) {
  922. if (ELF_ST_BIND(syms[k].st_info) != STB_GLOBAL)
  923. continue;
  924. if (strcmp(strtab + syms[k].st_name, name) != 0)
  925. continue;
  926. if (syms[k].st_shndx == SHN_UNDEF ||
  927. syms[k].st_shndx >= ehdr->e_shnum) {
  928. pr_debug("Symbol: %s has bad section index %d.\n",
  929. name, syms[k].st_shndx);
  930. return NULL;
  931. }
  932. /* Found the symbol we are looking for */
  933. return &syms[k];
  934. }
  935. }
  936. return NULL;
  937. }
  938. void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name)
  939. {
  940. struct purgatory_info *pi = &image->purgatory_info;
  941. const Elf_Sym *sym;
  942. Elf_Shdr *sechdr;
  943. sym = kexec_purgatory_find_symbol(pi, name);
  944. if (!sym)
  945. return ERR_PTR(-EINVAL);
  946. sechdr = &pi->sechdrs[sym->st_shndx];
  947. /*
  948. * Returns the address where symbol will finally be loaded after
  949. * kexec_load_segment()
  950. */
  951. return (void *)(sechdr->sh_addr + sym->st_value);
  952. }
  953. /*
  954. * Get or set value of a symbol. If "get_value" is true, symbol value is
  955. * returned in buf otherwise symbol value is set based on value in buf.
  956. */
  957. int kexec_purgatory_get_set_symbol(struct kimage *image, const char *name,
  958. void *buf, unsigned int size, bool get_value)
  959. {
  960. struct purgatory_info *pi = &image->purgatory_info;
  961. const Elf_Sym *sym;
  962. Elf_Shdr *sec;
  963. char *sym_buf;
  964. sym = kexec_purgatory_find_symbol(pi, name);
  965. if (!sym)
  966. return -EINVAL;
  967. if (sym->st_size != size) {
  968. pr_err("symbol %s size mismatch: expected %lu actual %u\n",
  969. name, (unsigned long)sym->st_size, size);
  970. return -EINVAL;
  971. }
  972. sec = pi->sechdrs + sym->st_shndx;
  973. if (sec->sh_type == SHT_NOBITS) {
  974. pr_err("symbol %s is in a bss section. Cannot %s\n", name,
  975. get_value ? "get" : "set");
  976. return -EINVAL;
  977. }
  978. sym_buf = (char *)pi->purgatory_buf + sec->sh_offset + sym->st_value;
  979. if (get_value)
  980. memcpy((void *)buf, sym_buf, size);
  981. else
  982. memcpy((void *)sym_buf, buf, size);
  983. return 0;
  984. }
  985. #endif /* CONFIG_ARCH_HAS_KEXEC_PURGATORY */
  986. int crash_exclude_mem_range(struct crash_mem *mem,
  987. unsigned long long mstart, unsigned long long mend)
  988. {
  989. int i, j;
  990. unsigned long long start, end, p_start, p_end;
  991. struct crash_mem_range temp_range = {0, 0};
  992. for (i = 0; i < mem->nr_ranges; i++) {
  993. start = mem->ranges[i].start;
  994. end = mem->ranges[i].end;
  995. p_start = mstart;
  996. p_end = mend;
  997. if (mstart > end || mend < start)
  998. continue;
  999. /* Truncate any area outside of range */
  1000. if (mstart < start)
  1001. p_start = start;
  1002. if (mend > end)
  1003. p_end = end;
  1004. /* Found completely overlapping range */
  1005. if (p_start == start && p_end == end) {
  1006. mem->ranges[i].start = 0;
  1007. mem->ranges[i].end = 0;
  1008. if (i < mem->nr_ranges - 1) {
  1009. /* Shift rest of the ranges to left */
  1010. for (j = i; j < mem->nr_ranges - 1; j++) {
  1011. mem->ranges[j].start =
  1012. mem->ranges[j+1].start;
  1013. mem->ranges[j].end =
  1014. mem->ranges[j+1].end;
  1015. }
  1016. /*
  1017. * Continue to check if there are another overlapping ranges
  1018. * from the current position because of shifting the above
  1019. * mem ranges.
  1020. */
  1021. i--;
  1022. mem->nr_ranges--;
  1023. continue;
  1024. }
  1025. mem->nr_ranges--;
  1026. return 0;
  1027. }
  1028. if (p_start > start && p_end < end) {
  1029. /* Split original range */
  1030. mem->ranges[i].end = p_start - 1;
  1031. temp_range.start = p_end + 1;
  1032. temp_range.end = end;
  1033. } else if (p_start != start)
  1034. mem->ranges[i].end = p_start - 1;
  1035. else
  1036. mem->ranges[i].start = p_end + 1;
  1037. break;
  1038. }
  1039. /* If a split happened, add the split to array */
  1040. if (!temp_range.end)
  1041. return 0;
  1042. /* Split happened */
  1043. if (i == mem->max_nr_ranges - 1)
  1044. return -ENOMEM;
  1045. /* Location where new range should go */
  1046. j = i + 1;
  1047. if (j < mem->nr_ranges) {
  1048. /* Move over all ranges one slot towards the end */
  1049. for (i = mem->nr_ranges - 1; i >= j; i--)
  1050. mem->ranges[i + 1] = mem->ranges[i];
  1051. }
  1052. mem->ranges[j].start = temp_range.start;
  1053. mem->ranges[j].end = temp_range.end;
  1054. mem->nr_ranges++;
  1055. return 0;
  1056. }
  1057. int crash_prepare_elf64_headers(struct crash_mem *mem, int kernel_map,
  1058. void **addr, unsigned long *sz)
  1059. {
  1060. Elf64_Ehdr *ehdr;
  1061. Elf64_Phdr *phdr;
  1062. unsigned long nr_cpus = num_possible_cpus(), nr_phdr, elf_sz;
  1063. unsigned char *buf;
  1064. unsigned int cpu, i;
  1065. unsigned long long notes_addr;
  1066. unsigned long mstart, mend;
  1067. /* extra phdr for vmcoreinfo ELF note */
  1068. nr_phdr = nr_cpus + 1;
  1069. nr_phdr += mem->nr_ranges;
  1070. /*
  1071. * kexec-tools creates an extra PT_LOAD phdr for kernel text mapping
  1072. * area (for example, ffffffff80000000 - ffffffffa0000000 on x86_64).
  1073. * I think this is required by tools like gdb. So same physical
  1074. * memory will be mapped in two ELF headers. One will contain kernel
  1075. * text virtual addresses and other will have __va(physical) addresses.
  1076. */
  1077. nr_phdr++;
  1078. elf_sz = sizeof(Elf64_Ehdr) + nr_phdr * sizeof(Elf64_Phdr);
  1079. elf_sz = ALIGN(elf_sz, ELF_CORE_HEADER_ALIGN);
  1080. buf = vzalloc(elf_sz);
  1081. if (!buf)
  1082. return -ENOMEM;
  1083. ehdr = (Elf64_Ehdr *)buf;
  1084. phdr = (Elf64_Phdr *)(ehdr + 1);
  1085. memcpy(ehdr->e_ident, ELFMAG, SELFMAG);
  1086. ehdr->e_ident[EI_CLASS] = ELFCLASS64;
  1087. ehdr->e_ident[EI_DATA] = ELFDATA2LSB;
  1088. ehdr->e_ident[EI_VERSION] = EV_CURRENT;
  1089. ehdr->e_ident[EI_OSABI] = ELF_OSABI;
  1090. memset(ehdr->e_ident + EI_PAD, 0, EI_NIDENT - EI_PAD);
  1091. ehdr->e_type = ET_CORE;
  1092. ehdr->e_machine = ELF_ARCH;
  1093. ehdr->e_version = EV_CURRENT;
  1094. ehdr->e_phoff = sizeof(Elf64_Ehdr);
  1095. ehdr->e_ehsize = sizeof(Elf64_Ehdr);
  1096. ehdr->e_phentsize = sizeof(Elf64_Phdr);
  1097. /* Prepare one phdr of type PT_NOTE for each present CPU */
  1098. for_each_present_cpu(cpu) {
  1099. phdr->p_type = PT_NOTE;
  1100. notes_addr = per_cpu_ptr_to_phys(per_cpu_ptr(crash_notes, cpu));
  1101. phdr->p_offset = phdr->p_paddr = notes_addr;
  1102. phdr->p_filesz = phdr->p_memsz = sizeof(note_buf_t);
  1103. (ehdr->e_phnum)++;
  1104. phdr++;
  1105. }
  1106. /* Prepare one PT_NOTE header for vmcoreinfo */
  1107. phdr->p_type = PT_NOTE;
  1108. phdr->p_offset = phdr->p_paddr = paddr_vmcoreinfo_note();
  1109. phdr->p_filesz = phdr->p_memsz = VMCOREINFO_NOTE_SIZE;
  1110. (ehdr->e_phnum)++;
  1111. phdr++;
  1112. /* Prepare PT_LOAD type program header for kernel text region */
  1113. if (kernel_map) {
  1114. phdr->p_type = PT_LOAD;
  1115. phdr->p_flags = PF_R|PF_W|PF_X;
  1116. phdr->p_vaddr = (unsigned long) _text;
  1117. phdr->p_filesz = phdr->p_memsz = _end - _text;
  1118. phdr->p_offset = phdr->p_paddr = __pa_symbol(_text);
  1119. ehdr->e_phnum++;
  1120. phdr++;
  1121. }
  1122. /* Go through all the ranges in mem->ranges[] and prepare phdr */
  1123. for (i = 0; i < mem->nr_ranges; i++) {
  1124. mstart = mem->ranges[i].start;
  1125. mend = mem->ranges[i].end;
  1126. phdr->p_type = PT_LOAD;
  1127. phdr->p_flags = PF_R|PF_W|PF_X;
  1128. phdr->p_offset = mstart;
  1129. phdr->p_paddr = mstart;
  1130. phdr->p_vaddr = (unsigned long) __va(mstart);
  1131. phdr->p_filesz = phdr->p_memsz = mend - mstart + 1;
  1132. phdr->p_align = 0;
  1133. ehdr->e_phnum++;
  1134. pr_debug("Crash PT_LOAD ELF header. phdr=%p vaddr=0x%llx, paddr=0x%llx, sz=0x%llx e_phnum=%d p_offset=0x%llx\n",
  1135. phdr, phdr->p_vaddr, phdr->p_paddr, phdr->p_filesz,
  1136. ehdr->e_phnum, phdr->p_offset);
  1137. phdr++;
  1138. }
  1139. *addr = buf;
  1140. *sz = elf_sz;
  1141. return 0;
  1142. }