vma.c 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Set up the VMAs to tell the VM about the vDSO.
  4. * Copyright 2007 Andi Kleen, SUSE Labs.
  5. */
  6. /*
  7. * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
  8. */
  9. #include <linux/mm.h>
  10. #include <linux/err.h>
  11. #include <linux/sched.h>
  12. #include <linux/slab.h>
  13. #include <linux/init.h>
  14. #include <linux/linkage.h>
  15. #include <linux/random.h>
  16. #include <linux/elf.h>
  17. #include <asm/cacheflush.h>
  18. #include <asm/spitfire.h>
  19. #include <asm/vdso.h>
  20. #include <asm/vvar.h>
  21. #include <asm/page.h>
  22. unsigned int __read_mostly vdso_enabled = 1;
  23. static struct vm_special_mapping vvar_mapping = {
  24. .name = "[vvar]"
  25. };
  26. #ifdef CONFIG_SPARC64
  27. static struct vm_special_mapping vdso_mapping64 = {
  28. .name = "[vdso]"
  29. };
  30. #endif
  31. #ifdef CONFIG_COMPAT
  32. static struct vm_special_mapping vdso_mapping32 = {
  33. .name = "[vdso]"
  34. };
  35. #endif
  36. struct vvar_data *vvar_data;
  37. struct vdso_elfinfo32 {
  38. Elf32_Ehdr *hdr;
  39. Elf32_Sym *dynsym;
  40. unsigned long dynsymsize;
  41. const char *dynstr;
  42. unsigned long text;
  43. };
  44. struct vdso_elfinfo64 {
  45. Elf64_Ehdr *hdr;
  46. Elf64_Sym *dynsym;
  47. unsigned long dynsymsize;
  48. const char *dynstr;
  49. unsigned long text;
  50. };
  51. struct vdso_elfinfo {
  52. union {
  53. struct vdso_elfinfo32 elf32;
  54. struct vdso_elfinfo64 elf64;
  55. } u;
  56. };
  57. static void *one_section64(struct vdso_elfinfo64 *e, const char *name,
  58. unsigned long *size)
  59. {
  60. const char *snames;
  61. Elf64_Shdr *shdrs;
  62. unsigned int i;
  63. shdrs = (void *)e->hdr + e->hdr->e_shoff;
  64. snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset;
  65. for (i = 1; i < e->hdr->e_shnum; i++) {
  66. if (!strcmp(snames+shdrs[i].sh_name, name)) {
  67. if (size)
  68. *size = shdrs[i].sh_size;
  69. return (void *)e->hdr + shdrs[i].sh_offset;
  70. }
  71. }
  72. return NULL;
  73. }
  74. static int find_sections64(const struct vdso_image *image, struct vdso_elfinfo *_e)
  75. {
  76. struct vdso_elfinfo64 *e = &_e->u.elf64;
  77. e->hdr = image->data;
  78. e->dynsym = one_section64(e, ".dynsym", &e->dynsymsize);
  79. e->dynstr = one_section64(e, ".dynstr", NULL);
  80. if (!e->dynsym || !e->dynstr) {
  81. pr_err("VDSO64: Missing symbol sections.\n");
  82. return -ENODEV;
  83. }
  84. return 0;
  85. }
  86. static Elf64_Sym *find_sym64(const struct vdso_elfinfo64 *e, const char *name)
  87. {
  88. unsigned int i;
  89. for (i = 0; i < (e->dynsymsize / sizeof(Elf64_Sym)); i++) {
  90. Elf64_Sym *s = &e->dynsym[i];
  91. if (s->st_name == 0)
  92. continue;
  93. if (!strcmp(e->dynstr + s->st_name, name))
  94. return s;
  95. }
  96. return NULL;
  97. }
  98. static int patchsym64(struct vdso_elfinfo *_e, const char *orig,
  99. const char *new)
  100. {
  101. struct vdso_elfinfo64 *e = &_e->u.elf64;
  102. Elf64_Sym *osym = find_sym64(e, orig);
  103. Elf64_Sym *nsym = find_sym64(e, new);
  104. if (!nsym || !osym) {
  105. pr_err("VDSO64: Missing symbols.\n");
  106. return -ENODEV;
  107. }
  108. osym->st_value = nsym->st_value;
  109. osym->st_size = nsym->st_size;
  110. osym->st_info = nsym->st_info;
  111. osym->st_other = nsym->st_other;
  112. osym->st_shndx = nsym->st_shndx;
  113. return 0;
  114. }
  115. static void *one_section32(struct vdso_elfinfo32 *e, const char *name,
  116. unsigned long *size)
  117. {
  118. const char *snames;
  119. Elf32_Shdr *shdrs;
  120. unsigned int i;
  121. shdrs = (void *)e->hdr + e->hdr->e_shoff;
  122. snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset;
  123. for (i = 1; i < e->hdr->e_shnum; i++) {
  124. if (!strcmp(snames+shdrs[i].sh_name, name)) {
  125. if (size)
  126. *size = shdrs[i].sh_size;
  127. return (void *)e->hdr + shdrs[i].sh_offset;
  128. }
  129. }
  130. return NULL;
  131. }
  132. static int find_sections32(const struct vdso_image *image, struct vdso_elfinfo *_e)
  133. {
  134. struct vdso_elfinfo32 *e = &_e->u.elf32;
  135. e->hdr = image->data;
  136. e->dynsym = one_section32(e, ".dynsym", &e->dynsymsize);
  137. e->dynstr = one_section32(e, ".dynstr", NULL);
  138. if (!e->dynsym || !e->dynstr) {
  139. pr_err("VDSO32: Missing symbol sections.\n");
  140. return -ENODEV;
  141. }
  142. return 0;
  143. }
  144. static Elf32_Sym *find_sym32(const struct vdso_elfinfo32 *e, const char *name)
  145. {
  146. unsigned int i;
  147. for (i = 0; i < (e->dynsymsize / sizeof(Elf32_Sym)); i++) {
  148. Elf32_Sym *s = &e->dynsym[i];
  149. if (s->st_name == 0)
  150. continue;
  151. if (!strcmp(e->dynstr + s->st_name, name))
  152. return s;
  153. }
  154. return NULL;
  155. }
  156. static int patchsym32(struct vdso_elfinfo *_e, const char *orig,
  157. const char *new)
  158. {
  159. struct vdso_elfinfo32 *e = &_e->u.elf32;
  160. Elf32_Sym *osym = find_sym32(e, orig);
  161. Elf32_Sym *nsym = find_sym32(e, new);
  162. if (!nsym || !osym) {
  163. pr_err("VDSO32: Missing symbols.\n");
  164. return -ENODEV;
  165. }
  166. osym->st_value = nsym->st_value;
  167. osym->st_size = nsym->st_size;
  168. osym->st_info = nsym->st_info;
  169. osym->st_other = nsym->st_other;
  170. osym->st_shndx = nsym->st_shndx;
  171. return 0;
  172. }
  173. static int find_sections(const struct vdso_image *image, struct vdso_elfinfo *e,
  174. bool elf64)
  175. {
  176. if (elf64)
  177. return find_sections64(image, e);
  178. else
  179. return find_sections32(image, e);
  180. }
  181. static int patch_one_symbol(struct vdso_elfinfo *e, const char *orig,
  182. const char *new_target, bool elf64)
  183. {
  184. if (elf64)
  185. return patchsym64(e, orig, new_target);
  186. else
  187. return patchsym32(e, orig, new_target);
  188. }
  189. static int stick_patch(const struct vdso_image *image, struct vdso_elfinfo *e, bool elf64)
  190. {
  191. int err;
  192. err = find_sections(image, e, elf64);
  193. if (err)
  194. return err;
  195. err = patch_one_symbol(e,
  196. "__vdso_gettimeofday",
  197. "__vdso_gettimeofday_stick", elf64);
  198. if (err)
  199. return err;
  200. return patch_one_symbol(e,
  201. "__vdso_clock_gettime",
  202. "__vdso_clock_gettime_stick", elf64);
  203. return 0;
  204. }
  205. /*
  206. * Allocate pages for the vdso and vvar, and copy in the vdso text from the
  207. * kernel image.
  208. */
  209. int __init init_vdso_image(const struct vdso_image *image,
  210. struct vm_special_mapping *vdso_mapping, bool elf64)
  211. {
  212. int cnpages = (image->size) / PAGE_SIZE;
  213. struct page *dp, **dpp = NULL;
  214. struct page *cp, **cpp = NULL;
  215. struct vdso_elfinfo ei;
  216. int i, dnpages = 0;
  217. if (tlb_type != spitfire) {
  218. int err = stick_patch(image, &ei, elf64);
  219. if (err)
  220. return err;
  221. }
  222. /*
  223. * First, the vdso text. This is initialied data, an integral number of
  224. * pages long.
  225. */
  226. if (WARN_ON(image->size % PAGE_SIZE != 0))
  227. goto oom;
  228. cpp = kcalloc(cnpages, sizeof(struct page *), GFP_KERNEL);
  229. vdso_mapping->pages = cpp;
  230. if (!cpp)
  231. goto oom;
  232. for (i = 0; i < cnpages; i++) {
  233. cp = alloc_page(GFP_KERNEL);
  234. if (!cp)
  235. goto oom;
  236. cpp[i] = cp;
  237. copy_page(page_address(cp), image->data + i * PAGE_SIZE);
  238. }
  239. /*
  240. * Now the vvar page. This is uninitialized data.
  241. */
  242. if (vvar_data == NULL) {
  243. dnpages = (sizeof(struct vvar_data) / PAGE_SIZE) + 1;
  244. if (WARN_ON(dnpages != 1))
  245. goto oom;
  246. dpp = kcalloc(dnpages, sizeof(struct page *), GFP_KERNEL);
  247. vvar_mapping.pages = dpp;
  248. if (!dpp)
  249. goto oom;
  250. dp = alloc_page(GFP_KERNEL);
  251. if (!dp)
  252. goto oom;
  253. dpp[0] = dp;
  254. vvar_data = page_address(dp);
  255. memset(vvar_data, 0, PAGE_SIZE);
  256. vvar_data->seq = 0;
  257. }
  258. return 0;
  259. oom:
  260. if (cpp != NULL) {
  261. for (i = 0; i < cnpages; i++) {
  262. if (cpp[i] != NULL)
  263. __free_page(cpp[i]);
  264. }
  265. kfree(cpp);
  266. vdso_mapping->pages = NULL;
  267. }
  268. if (dpp != NULL) {
  269. for (i = 0; i < dnpages; i++) {
  270. if (dpp[i] != NULL)
  271. __free_page(dpp[i]);
  272. }
  273. kfree(dpp);
  274. vvar_mapping.pages = NULL;
  275. }
  276. pr_warn("Cannot allocate vdso\n");
  277. vdso_enabled = 0;
  278. return -ENOMEM;
  279. }
  280. static int __init init_vdso(void)
  281. {
  282. int err = 0;
  283. #ifdef CONFIG_SPARC64
  284. err = init_vdso_image(&vdso_image_64_builtin, &vdso_mapping64, true);
  285. if (err)
  286. return err;
  287. #endif
  288. #ifdef CONFIG_COMPAT
  289. err = init_vdso_image(&vdso_image_32_builtin, &vdso_mapping32, false);
  290. #endif
  291. return err;
  292. }
  293. subsys_initcall(init_vdso);
  294. struct linux_binprm;
  295. /* Shuffle the vdso up a bit, randomly. */
  296. static unsigned long vdso_addr(unsigned long start, unsigned int len)
  297. {
  298. unsigned int offset;
  299. /* This loses some more bits than a modulo, but is cheaper */
  300. offset = get_random_int() & (PTRS_PER_PTE - 1);
  301. return start + (offset << PAGE_SHIFT);
  302. }
  303. static int map_vdso(const struct vdso_image *image,
  304. struct vm_special_mapping *vdso_mapping)
  305. {
  306. struct mm_struct *mm = current->mm;
  307. struct vm_area_struct *vma;
  308. unsigned long text_start, addr = 0;
  309. int ret = 0;
  310. mmap_write_lock(mm);
  311. /*
  312. * First, get an unmapped region: then randomize it, and make sure that
  313. * region is free.
  314. */
  315. if (current->flags & PF_RANDOMIZE) {
  316. addr = get_unmapped_area(NULL, 0,
  317. image->size - image->sym_vvar_start,
  318. 0, 0);
  319. if (IS_ERR_VALUE(addr)) {
  320. ret = addr;
  321. goto up_fail;
  322. }
  323. addr = vdso_addr(addr, image->size - image->sym_vvar_start);
  324. }
  325. addr = get_unmapped_area(NULL, addr,
  326. image->size - image->sym_vvar_start, 0, 0);
  327. if (IS_ERR_VALUE(addr)) {
  328. ret = addr;
  329. goto up_fail;
  330. }
  331. text_start = addr - image->sym_vvar_start;
  332. current->mm->context.vdso = (void __user *)text_start;
  333. /*
  334. * MAYWRITE to allow gdb to COW and set breakpoints
  335. */
  336. vma = _install_special_mapping(mm,
  337. text_start,
  338. image->size,
  339. VM_READ|VM_EXEC|
  340. VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
  341. vdso_mapping);
  342. if (IS_ERR(vma)) {
  343. ret = PTR_ERR(vma);
  344. goto up_fail;
  345. }
  346. vma = _install_special_mapping(mm,
  347. addr,
  348. -image->sym_vvar_start,
  349. VM_READ|VM_MAYREAD,
  350. &vvar_mapping);
  351. if (IS_ERR(vma)) {
  352. ret = PTR_ERR(vma);
  353. do_munmap(mm, text_start, image->size, NULL);
  354. }
  355. up_fail:
  356. if (ret)
  357. current->mm->context.vdso = NULL;
  358. mmap_write_unlock(mm);
  359. return ret;
  360. }
  361. int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
  362. {
  363. if (!vdso_enabled)
  364. return 0;
  365. #if defined CONFIG_COMPAT
  366. if (!(is_32bit_task()))
  367. return map_vdso(&vdso_image_64_builtin, &vdso_mapping64);
  368. else
  369. return map_vdso(&vdso_image_32_builtin, &vdso_mapping32);
  370. #else
  371. return map_vdso(&vdso_image_64_builtin, &vdso_mapping64);
  372. #endif
  373. }
  374. static __init int vdso_setup(char *s)
  375. {
  376. int err;
  377. unsigned long val;
  378. err = kstrtoul(s, 10, &val);
  379. if (err)
  380. return err;
  381. vdso_enabled = val;
  382. return 0;
  383. }
  384. __setup("vdso=", vdso_setup);