vdso.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * VDSO implementations.
  4. *
  5. * Copyright (C) 2012 ARM Limited
  6. *
  7. * Author: Will Deacon <will.deacon@arm.com>
  8. */
  9. #include <linux/cache.h>
  10. #include <linux/clocksource.h>
  11. #include <linux/elf.h>
  12. #include <linux/err.h>
  13. #include <linux/errno.h>
  14. #include <linux/gfp.h>
  15. #include <linux/kernel.h>
  16. #include <linux/mm.h>
  17. #include <linux/sched.h>
  18. #include <linux/signal.h>
  19. #include <linux/slab.h>
  20. #include <linux/time_namespace.h>
  21. #include <linux/timekeeper_internal.h>
  22. #include <linux/vmalloc.h>
  23. #include <vdso/datapage.h>
  24. #include <vdso/helpers.h>
  25. #include <vdso/vsyscall.h>
  26. #include <asm/cacheflush.h>
  27. #include <asm/signal32.h>
  28. #include <asm/vdso.h>
  29. extern char vdso_start[], vdso_end[];
  30. extern char vdso32_start[], vdso32_end[];
  31. enum vdso_abi {
  32. VDSO_ABI_AA64,
  33. VDSO_ABI_AA32,
  34. };
  35. enum vvar_pages {
  36. VVAR_DATA_PAGE_OFFSET,
  37. VVAR_TIMENS_PAGE_OFFSET,
  38. VVAR_NR_PAGES,
  39. };
  40. struct vdso_abi_info {
  41. const char *name;
  42. const char *vdso_code_start;
  43. const char *vdso_code_end;
  44. unsigned long vdso_pages;
  45. /* Data Mapping */
  46. struct vm_special_mapping *dm;
  47. /* Code Mapping */
  48. struct vm_special_mapping *cm;
  49. };
  50. static struct vdso_abi_info vdso_info[] __ro_after_init = {
  51. [VDSO_ABI_AA64] = {
  52. .name = "vdso",
  53. .vdso_code_start = vdso_start,
  54. .vdso_code_end = vdso_end,
  55. },
  56. #ifdef CONFIG_COMPAT_VDSO
  57. [VDSO_ABI_AA32] = {
  58. .name = "vdso32",
  59. .vdso_code_start = vdso32_start,
  60. .vdso_code_end = vdso32_end,
  61. },
  62. #endif /* CONFIG_COMPAT_VDSO */
  63. };
  64. /*
  65. * The vDSO data page.
  66. */
  67. static union {
  68. struct vdso_data data[CS_BASES];
  69. u8 page[PAGE_SIZE];
  70. } vdso_data_store __page_aligned_data;
  71. struct vdso_data *vdso_data = vdso_data_store.data;
  72. static int __vdso_remap(enum vdso_abi abi,
  73. const struct vm_special_mapping *sm,
  74. struct vm_area_struct *new_vma)
  75. {
  76. unsigned long new_size = new_vma->vm_end - new_vma->vm_start;
  77. unsigned long vdso_size = vdso_info[abi].vdso_code_end -
  78. vdso_info[abi].vdso_code_start;
  79. if (vdso_size != new_size)
  80. return -EINVAL;
  81. current->mm->context.vdso = (void *)new_vma->vm_start;
  82. return 0;
  83. }
  84. static int __vdso_init(enum vdso_abi abi)
  85. {
  86. int i;
  87. struct page **vdso_pagelist;
  88. unsigned long pfn;
  89. if (memcmp(vdso_info[abi].vdso_code_start, "\177ELF", 4)) {
  90. pr_err("vDSO is not a valid ELF object!\n");
  91. return -EINVAL;
  92. }
  93. vdso_info[abi].vdso_pages = (
  94. vdso_info[abi].vdso_code_end -
  95. vdso_info[abi].vdso_code_start) >>
  96. PAGE_SHIFT;
  97. vdso_pagelist = kcalloc(vdso_info[abi].vdso_pages,
  98. sizeof(struct page *),
  99. GFP_KERNEL);
  100. if (vdso_pagelist == NULL)
  101. return -ENOMEM;
  102. /* Grab the vDSO code pages. */
  103. pfn = sym_to_pfn(vdso_info[abi].vdso_code_start);
  104. for (i = 0; i < vdso_info[abi].vdso_pages; i++)
  105. vdso_pagelist[i] = pfn_to_page(pfn + i);
  106. vdso_info[abi].cm->pages = vdso_pagelist;
  107. return 0;
  108. }
  109. #ifdef CONFIG_TIME_NS
  110. struct vdso_data *arch_get_vdso_data(void *vvar_page)
  111. {
  112. return (struct vdso_data *)(vvar_page);
  113. }
  114. /*
  115. * The vvar mapping contains data for a specific time namespace, so when a task
  116. * changes namespace we must unmap its vvar data for the old namespace.
  117. * Subsequent faults will map in data for the new namespace.
  118. *
  119. * For more details see timens_setup_vdso_data().
  120. */
  121. int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
  122. {
  123. struct mm_struct *mm = task->mm;
  124. struct vm_area_struct *vma;
  125. mmap_read_lock(mm);
  126. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  127. unsigned long size = vma->vm_end - vma->vm_start;
  128. if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm))
  129. zap_page_range(vma, vma->vm_start, size);
  130. #ifdef CONFIG_COMPAT_VDSO
  131. if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA32].dm))
  132. zap_page_range(vma, vma->vm_start, size);
  133. #endif
  134. }
  135. mmap_read_unlock(mm);
  136. return 0;
  137. }
  138. static struct page *find_timens_vvar_page(struct vm_area_struct *vma)
  139. {
  140. if (likely(vma->vm_mm == current->mm))
  141. return current->nsproxy->time_ns->vvar_page;
  142. /*
  143. * VM_PFNMAP | VM_IO protect .fault() handler from being called
  144. * through interfaces like /proc/$pid/mem or
  145. * process_vm_{readv,writev}() as long as there's no .access()
  146. * in special_mapping_vmops.
  147. * For more details check_vma_flags() and __access_remote_vm()
  148. */
  149. WARN(1, "vvar_page accessed remotely");
  150. return NULL;
  151. }
  152. #else
  153. static struct page *find_timens_vvar_page(struct vm_area_struct *vma)
  154. {
  155. return NULL;
  156. }
  157. #endif
  158. static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
  159. struct vm_area_struct *vma, struct vm_fault *vmf)
  160. {
  161. struct page *timens_page = find_timens_vvar_page(vma);
  162. unsigned long pfn;
  163. switch (vmf->pgoff) {
  164. case VVAR_DATA_PAGE_OFFSET:
  165. if (timens_page)
  166. pfn = page_to_pfn(timens_page);
  167. else
  168. pfn = sym_to_pfn(vdso_data);
  169. break;
  170. #ifdef CONFIG_TIME_NS
  171. case VVAR_TIMENS_PAGE_OFFSET:
  172. /*
  173. * If a task belongs to a time namespace then a namespace
  174. * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
  175. * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
  176. * offset.
  177. * See also the comment near timens_setup_vdso_data().
  178. */
  179. if (!timens_page)
  180. return VM_FAULT_SIGBUS;
  181. pfn = sym_to_pfn(vdso_data);
  182. break;
  183. #endif /* CONFIG_TIME_NS */
  184. default:
  185. return VM_FAULT_SIGBUS;
  186. }
  187. return vmf_insert_pfn(vma, vmf->address, pfn);
  188. }
  189. static int vvar_mremap(const struct vm_special_mapping *sm,
  190. struct vm_area_struct *new_vma)
  191. {
  192. unsigned long new_size = new_vma->vm_end - new_vma->vm_start;
  193. if (new_size != VVAR_NR_PAGES * PAGE_SIZE)
  194. return -EINVAL;
  195. return 0;
  196. }
  197. static int __setup_additional_pages(enum vdso_abi abi,
  198. struct mm_struct *mm,
  199. struct linux_binprm *bprm,
  200. int uses_interp)
  201. {
  202. unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
  203. unsigned long gp_flags = 0;
  204. void *ret;
  205. BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES);
  206. vdso_text_len = vdso_info[abi].vdso_pages << PAGE_SHIFT;
  207. /* Be sure to map the data page */
  208. vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE;
  209. vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
  210. if (IS_ERR_VALUE(vdso_base)) {
  211. ret = ERR_PTR(vdso_base);
  212. goto up_fail;
  213. }
  214. ret = _install_special_mapping(mm, vdso_base, VVAR_NR_PAGES * PAGE_SIZE,
  215. VM_READ|VM_MAYREAD|VM_PFNMAP,
  216. vdso_info[abi].dm);
  217. if (IS_ERR(ret))
  218. goto up_fail;
  219. if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) && system_supports_bti())
  220. gp_flags = VM_ARM64_BTI;
  221. vdso_base += VVAR_NR_PAGES * PAGE_SIZE;
  222. mm->context.vdso = (void *)vdso_base;
  223. ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
  224. VM_READ|VM_EXEC|gp_flags|
  225. VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
  226. vdso_info[abi].cm);
  227. if (IS_ERR(ret))
  228. goto up_fail;
  229. return 0;
  230. up_fail:
  231. mm->context.vdso = NULL;
  232. return PTR_ERR(ret);
  233. }
  234. #ifdef CONFIG_COMPAT
  235. /*
  236. * Create and map the vectors page for AArch32 tasks.
  237. */
  238. static int aarch32_vdso_mremap(const struct vm_special_mapping *sm,
  239. struct vm_area_struct *new_vma)
  240. {
  241. return __vdso_remap(VDSO_ABI_AA32, sm, new_vma);
  242. }
  243. enum aarch32_map {
  244. AA32_MAP_VECTORS, /* kuser helpers */
  245. AA32_MAP_SIGPAGE,
  246. AA32_MAP_VVAR,
  247. AA32_MAP_VDSO,
  248. };
  249. static struct page *aarch32_vectors_page __ro_after_init;
  250. static struct page *aarch32_sig_page __ro_after_init;
  251. static struct vm_special_mapping aarch32_vdso_maps[] = {
  252. [AA32_MAP_VECTORS] = {
  253. .name = "[vectors]", /* ABI */
  254. .pages = &aarch32_vectors_page,
  255. },
  256. [AA32_MAP_SIGPAGE] = {
  257. .name = "[sigpage]", /* ABI */
  258. .pages = &aarch32_sig_page,
  259. },
  260. [AA32_MAP_VVAR] = {
  261. .name = "[vvar]",
  262. .fault = vvar_fault,
  263. .mremap = vvar_mremap,
  264. },
  265. [AA32_MAP_VDSO] = {
  266. .name = "[vdso]",
  267. .mremap = aarch32_vdso_mremap,
  268. },
  269. };
  270. static int aarch32_alloc_kuser_vdso_page(void)
  271. {
  272. extern char __kuser_helper_start[], __kuser_helper_end[];
  273. int kuser_sz = __kuser_helper_end - __kuser_helper_start;
  274. unsigned long vdso_page;
  275. if (!IS_ENABLED(CONFIG_KUSER_HELPERS))
  276. return 0;
  277. vdso_page = get_zeroed_page(GFP_ATOMIC);
  278. if (!vdso_page)
  279. return -ENOMEM;
  280. memcpy((void *)(vdso_page + 0x1000 - kuser_sz), __kuser_helper_start,
  281. kuser_sz);
  282. aarch32_vectors_page = virt_to_page(vdso_page);
  283. flush_dcache_page(aarch32_vectors_page);
  284. return 0;
  285. }
  286. static int aarch32_alloc_sigpage(void)
  287. {
  288. extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[];
  289. int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start;
  290. unsigned long sigpage;
  291. sigpage = get_zeroed_page(GFP_ATOMIC);
  292. if (!sigpage)
  293. return -ENOMEM;
  294. memcpy((void *)sigpage, __aarch32_sigret_code_start, sigret_sz);
  295. aarch32_sig_page = virt_to_page(sigpage);
  296. flush_dcache_page(aarch32_sig_page);
  297. return 0;
  298. }
  299. static int __aarch32_alloc_vdso_pages(void)
  300. {
  301. if (!IS_ENABLED(CONFIG_COMPAT_VDSO))
  302. return 0;
  303. vdso_info[VDSO_ABI_AA32].dm = &aarch32_vdso_maps[AA32_MAP_VVAR];
  304. vdso_info[VDSO_ABI_AA32].cm = &aarch32_vdso_maps[AA32_MAP_VDSO];
  305. return __vdso_init(VDSO_ABI_AA32);
  306. }
  307. static int __init aarch32_alloc_vdso_pages(void)
  308. {
  309. int ret;
  310. ret = __aarch32_alloc_vdso_pages();
  311. if (ret)
  312. return ret;
  313. ret = aarch32_alloc_sigpage();
  314. if (ret)
  315. return ret;
  316. return aarch32_alloc_kuser_vdso_page();
  317. }
  318. arch_initcall(aarch32_alloc_vdso_pages);
  319. static int aarch32_kuser_helpers_setup(struct mm_struct *mm)
  320. {
  321. void *ret;
  322. if (!IS_ENABLED(CONFIG_KUSER_HELPERS))
  323. return 0;
  324. /*
  325. * Avoid VM_MAYWRITE for compatibility with arch/arm/, where it's
  326. * not safe to CoW the page containing the CPU exception vectors.
  327. */
  328. ret = _install_special_mapping(mm, AARCH32_VECTORS_BASE, PAGE_SIZE,
  329. VM_READ | VM_EXEC |
  330. VM_MAYREAD | VM_MAYEXEC,
  331. &aarch32_vdso_maps[AA32_MAP_VECTORS]);
  332. return PTR_ERR_OR_ZERO(ret);
  333. }
  334. static int aarch32_sigreturn_setup(struct mm_struct *mm)
  335. {
  336. unsigned long addr;
  337. void *ret;
  338. addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0);
  339. if (IS_ERR_VALUE(addr)) {
  340. ret = ERR_PTR(addr);
  341. goto out;
  342. }
  343. /*
  344. * VM_MAYWRITE is required to allow gdb to Copy-on-Write and
  345. * set breakpoints.
  346. */
  347. ret = _install_special_mapping(mm, addr, PAGE_SIZE,
  348. VM_READ | VM_EXEC | VM_MAYREAD |
  349. VM_MAYWRITE | VM_MAYEXEC,
  350. &aarch32_vdso_maps[AA32_MAP_SIGPAGE]);
  351. if (IS_ERR(ret))
  352. goto out;
  353. mm->context.sigpage = (void *)addr;
  354. out:
  355. return PTR_ERR_OR_ZERO(ret);
  356. }
  357. int aarch32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
  358. {
  359. struct mm_struct *mm = current->mm;
  360. int ret;
  361. if (mmap_write_lock_killable(mm))
  362. return -EINTR;
  363. ret = aarch32_kuser_helpers_setup(mm);
  364. if (ret)
  365. goto out;
  366. if (IS_ENABLED(CONFIG_COMPAT_VDSO)) {
  367. ret = __setup_additional_pages(VDSO_ABI_AA32, mm, bprm,
  368. uses_interp);
  369. if (ret)
  370. goto out;
  371. }
  372. ret = aarch32_sigreturn_setup(mm);
  373. out:
  374. mmap_write_unlock(mm);
  375. return ret;
  376. }
  377. #endif /* CONFIG_COMPAT */
  378. static int vdso_mremap(const struct vm_special_mapping *sm,
  379. struct vm_area_struct *new_vma)
  380. {
  381. return __vdso_remap(VDSO_ABI_AA64, sm, new_vma);
  382. }
  383. enum aarch64_map {
  384. AA64_MAP_VVAR,
  385. AA64_MAP_VDSO,
  386. };
  387. static struct vm_special_mapping aarch64_vdso_maps[] __ro_after_init = {
  388. [AA64_MAP_VVAR] = {
  389. .name = "[vvar]",
  390. .fault = vvar_fault,
  391. .mremap = vvar_mremap,
  392. },
  393. [AA64_MAP_VDSO] = {
  394. .name = "[vdso]",
  395. .mremap = vdso_mremap,
  396. },
  397. };
  398. static int __init vdso_init(void)
  399. {
  400. vdso_info[VDSO_ABI_AA64].dm = &aarch64_vdso_maps[AA64_MAP_VVAR];
  401. vdso_info[VDSO_ABI_AA64].cm = &aarch64_vdso_maps[AA64_MAP_VDSO];
  402. return __vdso_init(VDSO_ABI_AA64);
  403. }
  404. arch_initcall(vdso_init);
  405. int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
  406. {
  407. struct mm_struct *mm = current->mm;
  408. int ret;
  409. if (mmap_write_lock_killable(mm))
  410. return -EINTR;
  411. ret = __setup_additional_pages(VDSO_ABI_AA64, mm, bprm, uses_interp);
  412. mmap_write_unlock(mm);
  413. return ret;
  414. }