tlb.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
  4. */
  5. #include <linux/mm.h>
  6. #include <linux/module.h>
  7. #include <linux/sched/signal.h>
  8. #include <asm/tlbflush.h>
  9. #include <as-layout.h>
  10. #include <mem_user.h>
  11. #include <os.h>
  12. #include <skas.h>
  13. #include <kern_util.h>
  14. struct host_vm_change {
  15. struct host_vm_op {
  16. enum { NONE, MMAP, MUNMAP, MPROTECT } type;
  17. union {
  18. struct {
  19. unsigned long addr;
  20. unsigned long len;
  21. unsigned int prot;
  22. int fd;
  23. __u64 offset;
  24. } mmap;
  25. struct {
  26. unsigned long addr;
  27. unsigned long len;
  28. } munmap;
  29. struct {
  30. unsigned long addr;
  31. unsigned long len;
  32. unsigned int prot;
  33. } mprotect;
  34. } u;
  35. } ops[1];
  36. int userspace;
  37. int index;
  38. struct mm_struct *mm;
  39. void *data;
  40. int force;
  41. };
  42. #define INIT_HVC(mm, force, userspace) \
  43. ((struct host_vm_change) \
  44. { .ops = { { .type = NONE } }, \
  45. .mm = mm, \
  46. .data = NULL, \
  47. .userspace = userspace, \
  48. .index = 0, \
  49. .force = force })
  50. static void report_enomem(void)
  51. {
  52. printk(KERN_ERR "UML ran out of memory on the host side! "
  53. "This can happen due to a memory limitation or "
  54. "vm.max_map_count has been reached.\n");
  55. }
  56. static int do_ops(struct host_vm_change *hvc, int end,
  57. int finished)
  58. {
  59. struct host_vm_op *op;
  60. int i, ret = 0;
  61. for (i = 0; i < end && !ret; i++) {
  62. op = &hvc->ops[i];
  63. switch (op->type) {
  64. case MMAP:
  65. if (hvc->userspace)
  66. ret = map(&hvc->mm->context.id, op->u.mmap.addr,
  67. op->u.mmap.len, op->u.mmap.prot,
  68. op->u.mmap.fd,
  69. op->u.mmap.offset, finished,
  70. &hvc->data);
  71. else
  72. map_memory(op->u.mmap.addr, op->u.mmap.offset,
  73. op->u.mmap.len, 1, 1, 1);
  74. break;
  75. case MUNMAP:
  76. if (hvc->userspace)
  77. ret = unmap(&hvc->mm->context.id,
  78. op->u.munmap.addr,
  79. op->u.munmap.len, finished,
  80. &hvc->data);
  81. else
  82. ret = os_unmap_memory(
  83. (void *) op->u.munmap.addr,
  84. op->u.munmap.len);
  85. break;
  86. case MPROTECT:
  87. if (hvc->userspace)
  88. ret = protect(&hvc->mm->context.id,
  89. op->u.mprotect.addr,
  90. op->u.mprotect.len,
  91. op->u.mprotect.prot,
  92. finished, &hvc->data);
  93. else
  94. ret = os_protect_memory(
  95. (void *) op->u.mprotect.addr,
  96. op->u.mprotect.len,
  97. 1, 1, 1);
  98. break;
  99. default:
  100. printk(KERN_ERR "Unknown op type %d in do_ops\n",
  101. op->type);
  102. BUG();
  103. break;
  104. }
  105. }
  106. if (ret == -ENOMEM)
  107. report_enomem();
  108. return ret;
  109. }
  110. static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
  111. unsigned int prot, struct host_vm_change *hvc)
  112. {
  113. __u64 offset;
  114. struct host_vm_op *last;
  115. int fd = -1, ret = 0;
  116. if (virt + len > STUB_START && virt < STUB_END)
  117. return -EINVAL;
  118. if (hvc->userspace)
  119. fd = phys_mapping(phys, &offset);
  120. else
  121. offset = phys;
  122. if (hvc->index != 0) {
  123. last = &hvc->ops[hvc->index - 1];
  124. if ((last->type == MMAP) &&
  125. (last->u.mmap.addr + last->u.mmap.len == virt) &&
  126. (last->u.mmap.prot == prot) && (last->u.mmap.fd == fd) &&
  127. (last->u.mmap.offset + last->u.mmap.len == offset)) {
  128. last->u.mmap.len += len;
  129. return 0;
  130. }
  131. }
  132. if (hvc->index == ARRAY_SIZE(hvc->ops)) {
  133. ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
  134. hvc->index = 0;
  135. }
  136. hvc->ops[hvc->index++] = ((struct host_vm_op)
  137. { .type = MMAP,
  138. .u = { .mmap = { .addr = virt,
  139. .len = len,
  140. .prot = prot,
  141. .fd = fd,
  142. .offset = offset }
  143. } });
  144. return ret;
  145. }
  146. static int add_munmap(unsigned long addr, unsigned long len,
  147. struct host_vm_change *hvc)
  148. {
  149. struct host_vm_op *last;
  150. int ret = 0;
  151. if (addr + len > STUB_START && addr < STUB_END)
  152. return -EINVAL;
  153. if (hvc->index != 0) {
  154. last = &hvc->ops[hvc->index - 1];
  155. if ((last->type == MUNMAP) &&
  156. (last->u.munmap.addr + last->u.mmap.len == addr)) {
  157. last->u.munmap.len += len;
  158. return 0;
  159. }
  160. }
  161. if (hvc->index == ARRAY_SIZE(hvc->ops)) {
  162. ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
  163. hvc->index = 0;
  164. }
  165. hvc->ops[hvc->index++] = ((struct host_vm_op)
  166. { .type = MUNMAP,
  167. .u = { .munmap = { .addr = addr,
  168. .len = len } } });
  169. return ret;
  170. }
  171. static int add_mprotect(unsigned long addr, unsigned long len,
  172. unsigned int prot, struct host_vm_change *hvc)
  173. {
  174. struct host_vm_op *last;
  175. int ret = 0;
  176. if (addr + len > STUB_START && addr < STUB_END)
  177. return -EINVAL;
  178. if (hvc->index != 0) {
  179. last = &hvc->ops[hvc->index - 1];
  180. if ((last->type == MPROTECT) &&
  181. (last->u.mprotect.addr + last->u.mprotect.len == addr) &&
  182. (last->u.mprotect.prot == prot)) {
  183. last->u.mprotect.len += len;
  184. return 0;
  185. }
  186. }
  187. if (hvc->index == ARRAY_SIZE(hvc->ops)) {
  188. ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
  189. hvc->index = 0;
  190. }
  191. hvc->ops[hvc->index++] = ((struct host_vm_op)
  192. { .type = MPROTECT,
  193. .u = { .mprotect = { .addr = addr,
  194. .len = len,
  195. .prot = prot } } });
  196. return ret;
  197. }
  198. #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
  199. static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
  200. unsigned long end,
  201. struct host_vm_change *hvc)
  202. {
  203. pte_t *pte;
  204. int r, w, x, prot, ret = 0;
  205. pte = pte_offset_kernel(pmd, addr);
  206. do {
  207. if ((addr >= STUB_START) && (addr < STUB_END))
  208. continue;
  209. r = pte_read(*pte);
  210. w = pte_write(*pte);
  211. x = pte_exec(*pte);
  212. if (!pte_young(*pte)) {
  213. r = 0;
  214. w = 0;
  215. } else if (!pte_dirty(*pte))
  216. w = 0;
  217. prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
  218. (x ? UM_PROT_EXEC : 0));
  219. if (hvc->force || pte_newpage(*pte)) {
  220. if (pte_present(*pte)) {
  221. if (pte_newpage(*pte))
  222. ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
  223. PAGE_SIZE, prot, hvc);
  224. } else
  225. ret = add_munmap(addr, PAGE_SIZE, hvc);
  226. } else if (pte_newprot(*pte))
  227. ret = add_mprotect(addr, PAGE_SIZE, prot, hvc);
  228. *pte = pte_mkuptodate(*pte);
  229. } while (pte++, addr += PAGE_SIZE, ((addr < end) && !ret));
  230. return ret;
  231. }
  232. static inline int update_pmd_range(pud_t *pud, unsigned long addr,
  233. unsigned long end,
  234. struct host_vm_change *hvc)
  235. {
  236. pmd_t *pmd;
  237. unsigned long next;
  238. int ret = 0;
  239. pmd = pmd_offset(pud, addr);
  240. do {
  241. next = pmd_addr_end(addr, end);
  242. if (!pmd_present(*pmd)) {
  243. if (hvc->force || pmd_newpage(*pmd)) {
  244. ret = add_munmap(addr, next - addr, hvc);
  245. pmd_mkuptodate(*pmd);
  246. }
  247. }
  248. else ret = update_pte_range(pmd, addr, next, hvc);
  249. } while (pmd++, addr = next, ((addr < end) && !ret));
  250. return ret;
  251. }
  252. static inline int update_pud_range(p4d_t *p4d, unsigned long addr,
  253. unsigned long end,
  254. struct host_vm_change *hvc)
  255. {
  256. pud_t *pud;
  257. unsigned long next;
  258. int ret = 0;
  259. pud = pud_offset(p4d, addr);
  260. do {
  261. next = pud_addr_end(addr, end);
  262. if (!pud_present(*pud)) {
  263. if (hvc->force || pud_newpage(*pud)) {
  264. ret = add_munmap(addr, next - addr, hvc);
  265. pud_mkuptodate(*pud);
  266. }
  267. }
  268. else ret = update_pmd_range(pud, addr, next, hvc);
  269. } while (pud++, addr = next, ((addr < end) && !ret));
  270. return ret;
  271. }
  272. static inline int update_p4d_range(pgd_t *pgd, unsigned long addr,
  273. unsigned long end,
  274. struct host_vm_change *hvc)
  275. {
  276. p4d_t *p4d;
  277. unsigned long next;
  278. int ret = 0;
  279. p4d = p4d_offset(pgd, addr);
  280. do {
  281. next = p4d_addr_end(addr, end);
  282. if (!p4d_present(*p4d)) {
  283. if (hvc->force || p4d_newpage(*p4d)) {
  284. ret = add_munmap(addr, next - addr, hvc);
  285. p4d_mkuptodate(*p4d);
  286. }
  287. } else
  288. ret = update_pud_range(p4d, addr, next, hvc);
  289. } while (p4d++, addr = next, ((addr < end) && !ret));
  290. return ret;
  291. }
  292. void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
  293. unsigned long end_addr, int force)
  294. {
  295. pgd_t *pgd;
  296. struct host_vm_change hvc;
  297. unsigned long addr = start_addr, next;
  298. int ret = 0, userspace = 1;
  299. hvc = INIT_HVC(mm, force, userspace);
  300. pgd = pgd_offset(mm, addr);
  301. do {
  302. next = pgd_addr_end(addr, end_addr);
  303. if (!pgd_present(*pgd)) {
  304. if (force || pgd_newpage(*pgd)) {
  305. ret = add_munmap(addr, next - addr, &hvc);
  306. pgd_mkuptodate(*pgd);
  307. }
  308. } else
  309. ret = update_p4d_range(pgd, addr, next, &hvc);
  310. } while (pgd++, addr = next, ((addr < end_addr) && !ret));
  311. if (!ret)
  312. ret = do_ops(&hvc, hvc.index, 1);
  313. /* This is not an else because ret is modified above */
  314. if (ret) {
  315. struct mm_id *mm_idp = &current->mm->context.id;
  316. printk(KERN_ERR "fix_range_common: failed, killing current "
  317. "process: %d\n", task_tgid_vnr(current));
  318. mm_idp->kill = 1;
  319. }
  320. }
  321. static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
  322. {
  323. struct mm_struct *mm;
  324. pgd_t *pgd;
  325. p4d_t *p4d;
  326. pud_t *pud;
  327. pmd_t *pmd;
  328. pte_t *pte;
  329. unsigned long addr, last;
  330. int updated = 0, err = 0, force = 0, userspace = 0;
  331. struct host_vm_change hvc;
  332. mm = &init_mm;
  333. hvc = INIT_HVC(mm, force, userspace);
  334. for (addr = start; addr < end;) {
  335. pgd = pgd_offset(mm, addr);
  336. if (!pgd_present(*pgd)) {
  337. last = ADD_ROUND(addr, PGDIR_SIZE);
  338. if (last > end)
  339. last = end;
  340. if (pgd_newpage(*pgd)) {
  341. updated = 1;
  342. err = add_munmap(addr, last - addr, &hvc);
  343. if (err < 0)
  344. panic("munmap failed, errno = %d\n",
  345. -err);
  346. }
  347. addr = last;
  348. continue;
  349. }
  350. p4d = p4d_offset(pgd, addr);
  351. if (!p4d_present(*p4d)) {
  352. last = ADD_ROUND(addr, P4D_SIZE);
  353. if (last > end)
  354. last = end;
  355. if (p4d_newpage(*p4d)) {
  356. updated = 1;
  357. err = add_munmap(addr, last - addr, &hvc);
  358. if (err < 0)
  359. panic("munmap failed, errno = %d\n",
  360. -err);
  361. }
  362. addr = last;
  363. continue;
  364. }
  365. pud = pud_offset(p4d, addr);
  366. if (!pud_present(*pud)) {
  367. last = ADD_ROUND(addr, PUD_SIZE);
  368. if (last > end)
  369. last = end;
  370. if (pud_newpage(*pud)) {
  371. updated = 1;
  372. err = add_munmap(addr, last - addr, &hvc);
  373. if (err < 0)
  374. panic("munmap failed, errno = %d\n",
  375. -err);
  376. }
  377. addr = last;
  378. continue;
  379. }
  380. pmd = pmd_offset(pud, addr);
  381. if (!pmd_present(*pmd)) {
  382. last = ADD_ROUND(addr, PMD_SIZE);
  383. if (last > end)
  384. last = end;
  385. if (pmd_newpage(*pmd)) {
  386. updated = 1;
  387. err = add_munmap(addr, last - addr, &hvc);
  388. if (err < 0)
  389. panic("munmap failed, errno = %d\n",
  390. -err);
  391. }
  392. addr = last;
  393. continue;
  394. }
  395. pte = pte_offset_kernel(pmd, addr);
  396. if (!pte_present(*pte) || pte_newpage(*pte)) {
  397. updated = 1;
  398. err = add_munmap(addr, PAGE_SIZE, &hvc);
  399. if (err < 0)
  400. panic("munmap failed, errno = %d\n",
  401. -err);
  402. if (pte_present(*pte))
  403. err = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
  404. PAGE_SIZE, 0, &hvc);
  405. }
  406. else if (pte_newprot(*pte)) {
  407. updated = 1;
  408. err = add_mprotect(addr, PAGE_SIZE, 0, &hvc);
  409. }
  410. addr += PAGE_SIZE;
  411. }
  412. if (!err)
  413. err = do_ops(&hvc, hvc.index, 1);
  414. if (err < 0)
  415. panic("flush_tlb_kernel failed, errno = %d\n", err);
  416. return updated;
  417. }
  418. void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
  419. {
  420. pgd_t *pgd;
  421. p4d_t *p4d;
  422. pud_t *pud;
  423. pmd_t *pmd;
  424. pte_t *pte;
  425. struct mm_struct *mm = vma->vm_mm;
  426. void *flush = NULL;
  427. int r, w, x, prot, err = 0;
  428. struct mm_id *mm_id;
  429. address &= PAGE_MASK;
  430. if (address >= STUB_START && address < STUB_END)
  431. goto kill;
  432. pgd = pgd_offset(mm, address);
  433. if (!pgd_present(*pgd))
  434. goto kill;
  435. p4d = p4d_offset(pgd, address);
  436. if (!p4d_present(*p4d))
  437. goto kill;
  438. pud = pud_offset(p4d, address);
  439. if (!pud_present(*pud))
  440. goto kill;
  441. pmd = pmd_offset(pud, address);
  442. if (!pmd_present(*pmd))
  443. goto kill;
  444. pte = pte_offset_kernel(pmd, address);
  445. r = pte_read(*pte);
  446. w = pte_write(*pte);
  447. x = pte_exec(*pte);
  448. if (!pte_young(*pte)) {
  449. r = 0;
  450. w = 0;
  451. } else if (!pte_dirty(*pte)) {
  452. w = 0;
  453. }
  454. mm_id = &mm->context.id;
  455. prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
  456. (x ? UM_PROT_EXEC : 0));
  457. if (pte_newpage(*pte)) {
  458. if (pte_present(*pte)) {
  459. unsigned long long offset;
  460. int fd;
  461. fd = phys_mapping(pte_val(*pte) & PAGE_MASK, &offset);
  462. err = map(mm_id, address, PAGE_SIZE, prot, fd, offset,
  463. 1, &flush);
  464. }
  465. else err = unmap(mm_id, address, PAGE_SIZE, 1, &flush);
  466. }
  467. else if (pte_newprot(*pte))
  468. err = protect(mm_id, address, PAGE_SIZE, prot, 1, &flush);
  469. if (err) {
  470. if (err == -ENOMEM)
  471. report_enomem();
  472. goto kill;
  473. }
  474. *pte = pte_mkuptodate(*pte);
  475. return;
  476. kill:
  477. printk(KERN_ERR "Failed to flush page for address 0x%lx\n", address);
  478. force_sig(SIGKILL);
  479. }
  480. void flush_tlb_all(void)
  481. {
  482. /*
  483. * Don't bother flushing if this address space is about to be
  484. * destroyed.
  485. */
  486. if (atomic_read(&current->mm->mm_users) == 0)
  487. return;
  488. flush_tlb_mm(current->mm);
  489. }
  490. void flush_tlb_kernel_range(unsigned long start, unsigned long end)
  491. {
  492. flush_tlb_kernel_range_common(start, end);
  493. }
  494. void flush_tlb_kernel_vm(void)
  495. {
  496. flush_tlb_kernel_range_common(start_vm, end_vm);
  497. }
  498. void __flush_tlb_one(unsigned long addr)
  499. {
  500. flush_tlb_kernel_range_common(addr, addr + PAGE_SIZE);
  501. }
  502. static void fix_range(struct mm_struct *mm, unsigned long start_addr,
  503. unsigned long end_addr, int force)
  504. {
  505. /*
  506. * Don't bother flushing if this address space is about to be
  507. * destroyed.
  508. */
  509. if (atomic_read(&mm->mm_users) == 0)
  510. return;
  511. fix_range_common(mm, start_addr, end_addr, force);
  512. }
  513. void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
  514. unsigned long end)
  515. {
  516. if (vma->vm_mm == NULL)
  517. flush_tlb_kernel_range_common(start, end);
  518. else fix_range(vma->vm_mm, start, end, 0);
  519. }
  520. EXPORT_SYMBOL(flush_tlb_range);
  521. void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
  522. unsigned long end)
  523. {
  524. fix_range(mm, start, end, 0);
  525. }
  526. void flush_tlb_mm(struct mm_struct *mm)
  527. {
  528. struct vm_area_struct *vma = mm->mmap;
  529. while (vma != NULL) {
  530. fix_range(mm, vma->vm_start, vma->vm_end, 0);
  531. vma = vma->vm_next;
  532. }
  533. }
  534. void force_flush_all(void)
  535. {
  536. struct mm_struct *mm = current->mm;
  537. struct vm_area_struct *vma = mm->mmap;
  538. while (vma != NULL) {
  539. fix_range(mm, vma->vm_start, vma->vm_end, 1);
  540. vma = vma->vm_next;
  541. }
  542. }