mremap.c 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * mm/mremap.c
  4. *
  5. * (C) Copyright 1996 Linus Torvalds
  6. *
  7. * Address space accounting code <alan@lxorguk.ukuu.org.uk>
  8. * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
  9. */
  10. #include <linux/mm.h>
  11. #include <linux/hugetlb.h>
  12. #include <linux/shm.h>
  13. #include <linux/ksm.h>
  14. #include <linux/mman.h>
  15. #include <linux/swap.h>
  16. #include <linux/capability.h>
  17. #include <linux/fs.h>
  18. #include <linux/swapops.h>
  19. #include <linux/highmem.h>
  20. #include <linux/security.h>
  21. #include <linux/syscalls.h>
  22. #include <linux/mmu_notifier.h>
  23. #include <linux/uaccess.h>
  24. #include <linux/mm-arch-hooks.h>
  25. #include <linux/userfaultfd_k.h>
  26. #include <asm/cacheflush.h>
  27. #include <asm/tlbflush.h>
  28. #include "internal.h"
  29. static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr)
  30. {
  31. pgd_t *pgd;
  32. p4d_t *p4d;
  33. pud_t *pud;
  34. pgd = pgd_offset(mm, addr);
  35. if (pgd_none_or_clear_bad(pgd))
  36. return NULL;
  37. p4d = p4d_offset(pgd, addr);
  38. if (p4d_none_or_clear_bad(p4d))
  39. return NULL;
  40. pud = pud_offset(p4d, addr);
  41. if (pud_none_or_clear_bad(pud))
  42. return NULL;
  43. return pud;
  44. }
  45. static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
  46. {
  47. pud_t *pud;
  48. pmd_t *pmd;
  49. pud = get_old_pud(mm, addr);
  50. if (!pud)
  51. return NULL;
  52. pmd = pmd_offset(pud, addr);
  53. if (pmd_none(*pmd))
  54. return NULL;
  55. return pmd;
  56. }
  57. static pud_t *alloc_new_pud(struct mm_struct *mm, struct vm_area_struct *vma,
  58. unsigned long addr)
  59. {
  60. pgd_t *pgd;
  61. p4d_t *p4d;
  62. pgd = pgd_offset(mm, addr);
  63. p4d = p4d_alloc(mm, pgd, addr);
  64. if (!p4d)
  65. return NULL;
  66. return pud_alloc(mm, p4d, addr);
  67. }
  68. static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
  69. unsigned long addr)
  70. {
  71. pud_t *pud;
  72. pmd_t *pmd;
  73. pud = alloc_new_pud(mm, vma, addr);
  74. if (!pud)
  75. return NULL;
  76. pmd = pmd_alloc(mm, pud, addr);
  77. if (!pmd)
  78. return NULL;
  79. VM_BUG_ON(pmd_trans_huge(*pmd));
  80. return pmd;
  81. }
  82. static void take_rmap_locks(struct vm_area_struct *vma)
  83. {
  84. if (vma->vm_file)
  85. i_mmap_lock_write(vma->vm_file->f_mapping);
  86. if (vma->anon_vma)
  87. anon_vma_lock_write(vma->anon_vma);
  88. }
  89. static void drop_rmap_locks(struct vm_area_struct *vma)
  90. {
  91. if (vma->anon_vma)
  92. anon_vma_unlock_write(vma->anon_vma);
  93. if (vma->vm_file)
  94. i_mmap_unlock_write(vma->vm_file->f_mapping);
  95. }
  96. static pte_t move_soft_dirty_pte(pte_t pte)
  97. {
  98. /*
  99. * Set soft dirty bit so we can notice
  100. * in userspace the ptes were moved.
  101. */
  102. #ifdef CONFIG_MEM_SOFT_DIRTY
  103. if (pte_present(pte))
  104. pte = pte_mksoft_dirty(pte);
  105. else if (is_swap_pte(pte))
  106. pte = pte_swp_mksoft_dirty(pte);
  107. #endif
  108. return pte;
  109. }
  110. static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
  111. unsigned long old_addr, unsigned long old_end,
  112. struct vm_area_struct *new_vma, pmd_t *new_pmd,
  113. unsigned long new_addr, bool need_rmap_locks)
  114. {
  115. struct mm_struct *mm = vma->vm_mm;
  116. pte_t *old_pte, *new_pte, pte;
  117. spinlock_t *old_ptl, *new_ptl;
  118. bool force_flush = false;
  119. unsigned long len = old_end - old_addr;
  120. /*
  121. * When need_rmap_locks is true, we take the i_mmap_rwsem and anon_vma
  122. * locks to ensure that rmap will always observe either the old or the
  123. * new ptes. This is the easiest way to avoid races with
  124. * truncate_pagecache(), page migration, etc...
  125. *
  126. * When need_rmap_locks is false, we use other ways to avoid
  127. * such races:
  128. *
  129. * - During exec() shift_arg_pages(), we use a specially tagged vma
  130. * which rmap call sites look for using vma_is_temporary_stack().
  131. *
  132. * - During mremap(), new_vma is often known to be placed after vma
  133. * in rmap traversal order. This ensures rmap will always observe
  134. * either the old pte, or the new pte, or both (the page table locks
  135. * serialize access to individual ptes, but only rmap traversal
  136. * order guarantees that we won't miss both the old and new ptes).
  137. */
  138. if (need_rmap_locks)
  139. take_rmap_locks(vma);
  140. /*
  141. * We don't have to worry about the ordering of src and dst
  142. * pte locks because exclusive mmap_lock prevents deadlock.
  143. */
  144. old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
  145. new_pte = pte_offset_map(new_pmd, new_addr);
  146. new_ptl = pte_lockptr(mm, new_pmd);
  147. if (new_ptl != old_ptl)
  148. spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
  149. flush_tlb_batched_pending(vma->vm_mm);
  150. arch_enter_lazy_mmu_mode();
  151. for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE,
  152. new_pte++, new_addr += PAGE_SIZE) {
  153. if (pte_none(*old_pte))
  154. continue;
  155. pte = ptep_get_and_clear(mm, old_addr, old_pte);
  156. /*
  157. * If we are remapping a valid PTE, make sure
  158. * to flush TLB before we drop the PTL for the
  159. * PTE.
  160. *
  161. * NOTE! Both old and new PTL matter: the old one
  162. * for racing with page_mkclean(), the new one to
  163. * make sure the physical page stays valid until
  164. * the TLB entry for the old mapping has been
  165. * flushed.
  166. */
  167. if (pte_present(pte))
  168. force_flush = true;
  169. pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
  170. pte = move_soft_dirty_pte(pte);
  171. set_pte_at(mm, new_addr, new_pte, pte);
  172. }
  173. arch_leave_lazy_mmu_mode();
  174. if (force_flush)
  175. flush_tlb_range(vma, old_end - len, old_end);
  176. if (new_ptl != old_ptl)
  177. spin_unlock(new_ptl);
  178. pte_unmap(new_pte - 1);
  179. pte_unmap_unlock(old_pte - 1, old_ptl);
  180. if (need_rmap_locks)
  181. drop_rmap_locks(vma);
  182. }
  183. #ifdef CONFIG_HAVE_MOVE_PMD
  184. static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
  185. unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd)
  186. {
  187. spinlock_t *old_ptl, *new_ptl;
  188. struct mm_struct *mm = vma->vm_mm;
  189. pmd_t pmd;
  190. /*
  191. * The destination pmd shouldn't be established, free_pgtables()
  192. * should have released it.
  193. *
  194. * However, there's a case during execve() where we use mremap
  195. * to move the initial stack, and in that case the target area
  196. * may overlap the source area (always moving down).
  197. *
  198. * If everything is PMD-aligned, that works fine, as moving
  199. * each pmd down will clear the source pmd. But if we first
  200. * have a few 4kB-only pages that get moved down, and then
  201. * hit the "now the rest is PMD-aligned, let's do everything
  202. * one pmd at a time", we will still have the old (now empty
  203. * of any 4kB pages, but still there) PMD in the page table
  204. * tree.
  205. *
  206. * Warn on it once - because we really should try to figure
  207. * out how to do this better - but then say "I won't move
  208. * this pmd".
  209. *
  210. * One alternative might be to just unmap the target pmd at
  211. * this point, and verify that it really is empty. We'll see.
  212. */
  213. if (WARN_ON_ONCE(!pmd_none(*new_pmd)))
  214. return false;
  215. /*
  216. * We don't have to worry about the ordering of src and dst
  217. * ptlocks because exclusive mmap_lock prevents deadlock.
  218. */
  219. old_ptl = pmd_lock(vma->vm_mm, old_pmd);
  220. new_ptl = pmd_lockptr(mm, new_pmd);
  221. if (new_ptl != old_ptl)
  222. spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
  223. /* Clear the pmd */
  224. pmd = *old_pmd;
  225. pmd_clear(old_pmd);
  226. VM_BUG_ON(!pmd_none(*new_pmd));
  227. /* Set the new pmd */
  228. set_pmd_at(mm, new_addr, new_pmd, pmd);
  229. flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
  230. if (new_ptl != old_ptl)
  231. spin_unlock(new_ptl);
  232. spin_unlock(old_ptl);
  233. return true;
  234. }
  235. #else
  236. static inline bool move_normal_pmd(struct vm_area_struct *vma,
  237. unsigned long old_addr, unsigned long new_addr, pmd_t *old_pmd,
  238. pmd_t *new_pmd)
  239. {
  240. return false;
  241. }
  242. #endif
  243. #ifdef CONFIG_HAVE_MOVE_PUD
  244. static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr,
  245. unsigned long new_addr, pud_t *old_pud, pud_t *new_pud)
  246. {
  247. spinlock_t *old_ptl, *new_ptl;
  248. struct mm_struct *mm = vma->vm_mm;
  249. pud_t pud;
  250. /*
  251. * The destination pud shouldn't be established, free_pgtables()
  252. * should have released it.
  253. */
  254. if (WARN_ON_ONCE(!pud_none(*new_pud)))
  255. return false;
  256. /*
  257. * We don't have to worry about the ordering of src and dst
  258. * ptlocks because exclusive mmap_lock prevents deadlock.
  259. */
  260. old_ptl = pud_lock(vma->vm_mm, old_pud);
  261. new_ptl = pud_lockptr(mm, new_pud);
  262. if (new_ptl != old_ptl)
  263. spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
  264. /* Clear the pud */
  265. pud = *old_pud;
  266. pud_clear(old_pud);
  267. VM_BUG_ON(!pud_none(*new_pud));
  268. /* Set the new pud */
  269. set_pud_at(mm, new_addr, new_pud, pud);
  270. flush_tlb_range(vma, old_addr, old_addr + PUD_SIZE);
  271. if (new_ptl != old_ptl)
  272. spin_unlock(new_ptl);
  273. spin_unlock(old_ptl);
  274. return true;
  275. }
  276. #else
  277. static inline bool move_normal_pud(struct vm_area_struct *vma,
  278. unsigned long old_addr, unsigned long new_addr, pud_t *old_pud,
  279. pud_t *new_pud)
  280. {
  281. return false;
  282. }
  283. #endif
  284. enum pgt_entry {
  285. NORMAL_PMD,
  286. HPAGE_PMD,
  287. NORMAL_PUD,
  288. };
  289. /*
  290. * Returns an extent of the corresponding size for the pgt_entry specified if
  291. * valid. Else returns a smaller extent bounded by the end of the source and
  292. * destination pgt_entry.
  293. */
  294. static __always_inline unsigned long get_extent(enum pgt_entry entry,
  295. unsigned long old_addr, unsigned long old_end,
  296. unsigned long new_addr)
  297. {
  298. unsigned long next, extent, mask, size;
  299. switch (entry) {
  300. case HPAGE_PMD:
  301. case NORMAL_PMD:
  302. mask = PMD_MASK;
  303. size = PMD_SIZE;
  304. break;
  305. case NORMAL_PUD:
  306. mask = PUD_MASK;
  307. size = PUD_SIZE;
  308. break;
  309. default:
  310. BUILD_BUG();
  311. break;
  312. }
  313. next = (old_addr + size) & mask;
  314. /* even if next overflowed, extent below will be ok */
  315. extent = next - old_addr;
  316. if (extent > old_end - old_addr)
  317. extent = old_end - old_addr;
  318. next = (new_addr + size) & mask;
  319. if (extent > next - new_addr)
  320. extent = next - new_addr;
  321. return extent;
  322. }
  323. /*
  324. * Attempts to speedup the move by moving entry at the level corresponding to
  325. * pgt_entry. Returns true if the move was successful, else false.
  326. */
  327. static bool move_pgt_entry(enum pgt_entry entry, struct vm_area_struct *vma,
  328. unsigned long old_addr, unsigned long new_addr,
  329. void *old_entry, void *new_entry, bool need_rmap_locks)
  330. {
  331. bool moved = false;
  332. /* See comment in move_ptes() */
  333. if (need_rmap_locks)
  334. take_rmap_locks(vma);
  335. switch (entry) {
  336. case NORMAL_PMD:
  337. moved = move_normal_pmd(vma, old_addr, new_addr, old_entry,
  338. new_entry);
  339. break;
  340. case NORMAL_PUD:
  341. moved = move_normal_pud(vma, old_addr, new_addr, old_entry,
  342. new_entry);
  343. break;
  344. case HPAGE_PMD:
  345. moved = IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
  346. move_huge_pmd(vma, old_addr, new_addr, old_entry,
  347. new_entry);
  348. break;
  349. default:
  350. WARN_ON_ONCE(1);
  351. break;
  352. }
  353. if (need_rmap_locks)
  354. drop_rmap_locks(vma);
  355. return moved;
  356. }
  357. unsigned long move_page_tables(struct vm_area_struct *vma,
  358. unsigned long old_addr, struct vm_area_struct *new_vma,
  359. unsigned long new_addr, unsigned long len,
  360. bool need_rmap_locks)
  361. {
  362. unsigned long extent, old_end;
  363. struct mmu_notifier_range range;
  364. pmd_t *old_pmd, *new_pmd;
  365. if (!len)
  366. return 0;
  367. old_end = old_addr + len;
  368. flush_cache_range(vma, old_addr, old_end);
  369. mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
  370. old_addr, old_end);
  371. mmu_notifier_invalidate_range_start(&range);
  372. for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
  373. cond_resched();
  374. /*
  375. * If extent is PUD-sized try to speed up the move by moving at the
  376. * PUD level if possible.
  377. */
  378. extent = get_extent(NORMAL_PUD, old_addr, old_end, new_addr);
  379. if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) {
  380. pud_t *old_pud, *new_pud;
  381. old_pud = get_old_pud(vma->vm_mm, old_addr);
  382. if (!old_pud)
  383. continue;
  384. new_pud = alloc_new_pud(vma->vm_mm, vma, new_addr);
  385. if (!new_pud)
  386. break;
  387. if (move_pgt_entry(NORMAL_PUD, vma, old_addr, new_addr,
  388. old_pud, new_pud, true))
  389. continue;
  390. }
  391. extent = get_extent(NORMAL_PMD, old_addr, old_end, new_addr);
  392. old_pmd = get_old_pmd(vma->vm_mm, old_addr);
  393. if (!old_pmd)
  394. continue;
  395. new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr);
  396. if (!new_pmd)
  397. break;
  398. if (is_swap_pmd(*old_pmd) || pmd_trans_huge(*old_pmd) ||
  399. pmd_devmap(*old_pmd)) {
  400. if (extent == HPAGE_PMD_SIZE &&
  401. move_pgt_entry(HPAGE_PMD, vma, old_addr, new_addr,
  402. old_pmd, new_pmd, need_rmap_locks))
  403. continue;
  404. split_huge_pmd(vma, old_pmd, old_addr);
  405. if (pmd_trans_unstable(old_pmd))
  406. continue;
  407. } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PMD) &&
  408. extent == PMD_SIZE) {
  409. /*
  410. * If the extent is PMD-sized, try to speed the move by
  411. * moving at the PMD level if possible.
  412. */
  413. if (move_pgt_entry(NORMAL_PMD, vma, old_addr, new_addr,
  414. old_pmd, new_pmd, true))
  415. continue;
  416. }
  417. if (pte_alloc(new_vma->vm_mm, new_pmd))
  418. break;
  419. move_ptes(vma, old_pmd, old_addr, old_addr + extent, new_vma,
  420. new_pmd, new_addr, need_rmap_locks);
  421. }
  422. mmu_notifier_invalidate_range_end(&range);
  423. return len + old_addr - old_end; /* how much done */
  424. }
  425. static unsigned long move_vma(struct vm_area_struct *vma,
  426. unsigned long old_addr, unsigned long old_len,
  427. unsigned long new_len, unsigned long new_addr,
  428. bool *locked, unsigned long flags,
  429. struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap)
  430. {
  431. struct mm_struct *mm = vma->vm_mm;
  432. struct vm_area_struct *new_vma;
  433. unsigned long vm_flags = vma->vm_flags;
  434. unsigned long new_pgoff;
  435. unsigned long moved_len;
  436. unsigned long excess = 0;
  437. unsigned long hiwater_vm;
  438. int split = 0;
  439. int err;
  440. bool need_rmap_locks;
  441. /*
  442. * We'd prefer to avoid failure later on in do_munmap:
  443. * which may split one vma into three before unmapping.
  444. */
  445. if (mm->map_count >= sysctl_max_map_count - 3)
  446. return -ENOMEM;
  447. /*
  448. * Advise KSM to break any KSM pages in the area to be moved:
  449. * it would be confusing if they were to turn up at the new
  450. * location, where they happen to coincide with different KSM
  451. * pages recently unmapped. But leave vma->vm_flags as it was,
  452. * so KSM can come around to merge on vma and new_vma afterwards.
  453. */
  454. err = ksm_madvise(vma, old_addr, old_addr + old_len,
  455. MADV_UNMERGEABLE, &vm_flags);
  456. if (err)
  457. return err;
  458. new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
  459. new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff,
  460. &need_rmap_locks);
  461. if (!new_vma)
  462. return -ENOMEM;
  463. /* new_vma is returned protected by copy_vma, to prevent speculative
  464. * page fault to be done in the destination area before we move the pte.
  465. * Now, we must also protect the source VMA since we don't want pages
  466. * to be mapped in our back while we are copying the PTEs.
  467. */
  468. if (vma != new_vma)
  469. vm_write_begin(vma);
  470. moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len,
  471. need_rmap_locks);
  472. if (moved_len < old_len) {
  473. err = -ENOMEM;
  474. } else if (vma->vm_ops && vma->vm_ops->mremap) {
  475. err = vma->vm_ops->mremap(new_vma);
  476. }
  477. if (unlikely(err)) {
  478. /*
  479. * On error, move entries back from new area to old,
  480. * which will succeed since page tables still there,
  481. * and then proceed to unmap new area instead of old.
  482. */
  483. move_page_tables(new_vma, new_addr, vma, old_addr, moved_len,
  484. true);
  485. if (vma != new_vma)
  486. vm_write_end(vma);
  487. vma = new_vma;
  488. old_len = new_len;
  489. old_addr = new_addr;
  490. new_addr = err;
  491. } else {
  492. mremap_userfaultfd_prep(new_vma, uf);
  493. arch_remap(mm, old_addr, old_addr + old_len,
  494. new_addr, new_addr + new_len);
  495. if (vma != new_vma)
  496. vm_write_end(vma);
  497. }
  498. vm_write_end(new_vma);
  499. /* Conceal VM_ACCOUNT so old reservation is not undone */
  500. if (vm_flags & VM_ACCOUNT) {
  501. vma->vm_flags &= ~VM_ACCOUNT;
  502. excess = vma->vm_end - vma->vm_start - old_len;
  503. if (old_addr > vma->vm_start &&
  504. old_addr + old_len < vma->vm_end)
  505. split = 1;
  506. }
  507. /*
  508. * If we failed to move page tables we still do total_vm increment
  509. * since do_munmap() will decrement it by old_len == new_len.
  510. *
  511. * Since total_vm is about to be raised artificially high for a
  512. * moment, we need to restore high watermark afterwards: if stats
  513. * are taken meanwhile, total_vm and hiwater_vm appear too high.
  514. * If this were a serious issue, we'd add a flag to do_munmap().
  515. */
  516. hiwater_vm = mm->hiwater_vm;
  517. vm_stat_account(mm, vma->vm_flags, new_len >> PAGE_SHIFT);
  518. /* Tell pfnmap has moved from this vma */
  519. if (unlikely(vma->vm_flags & VM_PFNMAP))
  520. untrack_pfn_moved(vma);
  521. if (unlikely(!err && (flags & MREMAP_DONTUNMAP))) {
  522. if (vm_flags & VM_ACCOUNT) {
  523. /* Always put back VM_ACCOUNT since we won't unmap */
  524. vma->vm_flags |= VM_ACCOUNT;
  525. vm_acct_memory(new_len >> PAGE_SHIFT);
  526. }
  527. /*
  528. * VMAs can actually be merged back together in copy_vma
  529. * calling merge_vma. This can happen with anonymous vmas
  530. * which have not yet been faulted, so if we were to consider
  531. * this VMA split we'll end up adding VM_ACCOUNT on the
  532. * next VMA, which is completely unrelated if this VMA
  533. * was re-merged.
  534. */
  535. if (split && new_vma == vma)
  536. split = 0;
  537. /* We always clear VM_LOCKED[ONFAULT] on the old vma */
  538. vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
  539. /* Because we won't unmap we don't need to touch locked_vm */
  540. goto out;
  541. }
  542. if (do_munmap(mm, old_addr, old_len, uf_unmap) < 0) {
  543. /* OOM: unable to split vma, just get accounts right */
  544. vm_unacct_memory(excess >> PAGE_SHIFT);
  545. excess = 0;
  546. }
  547. if (vm_flags & VM_LOCKED) {
  548. mm->locked_vm += new_len >> PAGE_SHIFT;
  549. *locked = true;
  550. }
  551. out:
  552. mm->hiwater_vm = hiwater_vm;
  553. /* Restore VM_ACCOUNT if one or two pieces of vma left */
  554. if (excess) {
  555. vma->vm_flags |= VM_ACCOUNT;
  556. if (split)
  557. vma->vm_next->vm_flags |= VM_ACCOUNT;
  558. }
  559. return new_addr;
  560. }
  561. static struct vm_area_struct *vma_to_resize(unsigned long addr,
  562. unsigned long old_len, unsigned long new_len, unsigned long flags,
  563. unsigned long *p)
  564. {
  565. struct mm_struct *mm = current->mm;
  566. struct vm_area_struct *vma = find_vma(mm, addr);
  567. unsigned long pgoff;
  568. if (!vma || vma->vm_start > addr)
  569. return ERR_PTR(-EFAULT);
  570. /*
  571. * !old_len is a special case where an attempt is made to 'duplicate'
  572. * a mapping. This makes no sense for private mappings as it will
  573. * instead create a fresh/new mapping unrelated to the original. This
  574. * is contrary to the basic idea of mremap which creates new mappings
  575. * based on the original. There are no known use cases for this
  576. * behavior. As a result, fail such attempts.
  577. */
  578. if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) {
  579. pr_warn_once("%s (%d): attempted to duplicate a private mapping with mremap. This is not supported.\n", current->comm, current->pid);
  580. return ERR_PTR(-EINVAL);
  581. }
  582. if ((flags & MREMAP_DONTUNMAP) &&
  583. (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)))
  584. return ERR_PTR(-EINVAL);
  585. if (is_vm_hugetlb_page(vma))
  586. return ERR_PTR(-EINVAL);
  587. /* We can't remap across vm area boundaries */
  588. if (old_len > vma->vm_end - addr)
  589. return ERR_PTR(-EFAULT);
  590. if (new_len == old_len)
  591. return vma;
  592. /* Need to be careful about a growing mapping */
  593. pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
  594. pgoff += vma->vm_pgoff;
  595. if (pgoff + (new_len >> PAGE_SHIFT) < pgoff)
  596. return ERR_PTR(-EINVAL);
  597. if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))
  598. return ERR_PTR(-EFAULT);
  599. if (vma->vm_flags & VM_LOCKED) {
  600. unsigned long locked, lock_limit;
  601. locked = mm->locked_vm << PAGE_SHIFT;
  602. lock_limit = rlimit(RLIMIT_MEMLOCK);
  603. locked += new_len - old_len;
  604. if (locked > lock_limit && !capable(CAP_IPC_LOCK))
  605. return ERR_PTR(-EAGAIN);
  606. }
  607. if (!may_expand_vm(mm, vma->vm_flags,
  608. (new_len - old_len) >> PAGE_SHIFT))
  609. return ERR_PTR(-ENOMEM);
  610. if (vma->vm_flags & VM_ACCOUNT) {
  611. unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
  612. if (security_vm_enough_memory_mm(mm, charged))
  613. return ERR_PTR(-ENOMEM);
  614. *p = charged;
  615. }
  616. return vma;
  617. }
  618. static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
  619. unsigned long new_addr, unsigned long new_len, bool *locked,
  620. unsigned long flags, struct vm_userfaultfd_ctx *uf,
  621. struct list_head *uf_unmap_early,
  622. struct list_head *uf_unmap)
  623. {
  624. struct mm_struct *mm = current->mm;
  625. struct vm_area_struct *vma;
  626. unsigned long ret = -EINVAL;
  627. unsigned long charged = 0;
  628. unsigned long map_flags = 0;
  629. if (offset_in_page(new_addr))
  630. goto out;
  631. if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
  632. goto out;
  633. /* Ensure the old/new locations do not overlap */
  634. if (addr + old_len > new_addr && new_addr + new_len > addr)
  635. goto out;
  636. /*
  637. * move_vma() need us to stay 4 maps below the threshold, otherwise
  638. * it will bail out at the very beginning.
  639. * That is a problem if we have already unmaped the regions here
  640. * (new_addr, and old_addr), because userspace will not know the
  641. * state of the vma's after it gets -ENOMEM.
  642. * So, to avoid such scenario we can pre-compute if the whole
  643. * operation has high chances to success map-wise.
  644. * Worst-scenario case is when both vma's (new_addr and old_addr) get
  645. * split in 3 before unmaping it.
  646. * That means 2 more maps (1 for each) to the ones we already hold.
  647. * Check whether current map count plus 2 still leads us to 4 maps below
  648. * the threshold, otherwise return -ENOMEM here to be more safe.
  649. */
  650. if ((mm->map_count + 2) >= sysctl_max_map_count - 3)
  651. return -ENOMEM;
  652. if (flags & MREMAP_FIXED) {
  653. ret = do_munmap(mm, new_addr, new_len, uf_unmap_early);
  654. if (ret)
  655. goto out;
  656. }
  657. if (old_len >= new_len) {
  658. ret = do_munmap(mm, addr+new_len, old_len - new_len, uf_unmap);
  659. if (ret && old_len != new_len)
  660. goto out;
  661. old_len = new_len;
  662. }
  663. vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
  664. if (IS_ERR(vma)) {
  665. ret = PTR_ERR(vma);
  666. goto out;
  667. }
  668. /* MREMAP_DONTUNMAP expands by old_len since old_len == new_len */
  669. if (flags & MREMAP_DONTUNMAP &&
  670. !may_expand_vm(mm, vma->vm_flags, old_len >> PAGE_SHIFT)) {
  671. ret = -ENOMEM;
  672. goto out;
  673. }
  674. if (flags & MREMAP_FIXED)
  675. map_flags |= MAP_FIXED;
  676. if (vma->vm_flags & VM_MAYSHARE)
  677. map_flags |= MAP_SHARED;
  678. ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff +
  679. ((addr - vma->vm_start) >> PAGE_SHIFT),
  680. map_flags);
  681. if (IS_ERR_VALUE(ret))
  682. goto out1;
  683. /* We got a new mapping */
  684. if (!(flags & MREMAP_FIXED))
  685. new_addr = ret;
  686. ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf,
  687. uf_unmap);
  688. if (!(offset_in_page(ret)))
  689. goto out;
  690. out1:
  691. vm_unacct_memory(charged);
  692. out:
  693. return ret;
  694. }
  695. static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
  696. {
  697. unsigned long end = vma->vm_end + delta;
  698. if (end < vma->vm_end) /* overflow */
  699. return 0;
  700. if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */
  701. return 0;
  702. if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start,
  703. 0, MAP_FIXED) & ~PAGE_MASK)
  704. return 0;
  705. return 1;
  706. }
  707. /*
  708. * Expand (or shrink) an existing mapping, potentially moving it at the
  709. * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
  710. *
  711. * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
  712. * This option implies MREMAP_MAYMOVE.
  713. */
  714. SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
  715. unsigned long, new_len, unsigned long, flags,
  716. unsigned long, new_addr)
  717. {
  718. struct mm_struct *mm = current->mm;
  719. struct vm_area_struct *vma;
  720. unsigned long ret = -EINVAL;
  721. unsigned long charged = 0;
  722. bool locked = false;
  723. bool downgraded = false;
  724. struct vm_userfaultfd_ctx uf = NULL_VM_UFFD_CTX;
  725. LIST_HEAD(uf_unmap_early);
  726. LIST_HEAD(uf_unmap);
  727. /*
  728. * There is a deliberate asymmetry here: we strip the pointer tag
  729. * from the old address but leave the new address alone. This is
  730. * for consistency with mmap(), where we prevent the creation of
  731. * aliasing mappings in userspace by leaving the tag bits of the
  732. * mapping address intact. A non-zero tag will cause the subsequent
  733. * range checks to reject the address as invalid.
  734. *
  735. * See Documentation/arm64/tagged-address-abi.rst for more information.
  736. */
  737. addr = untagged_addr(addr);
  738. if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP))
  739. return ret;
  740. if (flags & MREMAP_FIXED && !(flags & MREMAP_MAYMOVE))
  741. return ret;
  742. /*
  743. * MREMAP_DONTUNMAP is always a move and it does not allow resizing
  744. * in the process.
  745. */
  746. if (flags & MREMAP_DONTUNMAP &&
  747. (!(flags & MREMAP_MAYMOVE) || old_len != new_len))
  748. return ret;
  749. if (offset_in_page(addr))
  750. return ret;
  751. old_len = PAGE_ALIGN(old_len);
  752. new_len = PAGE_ALIGN(new_len);
  753. /*
  754. * We allow a zero old-len as a special case
  755. * for DOS-emu "duplicate shm area" thing. But
  756. * a zero new-len is nonsensical.
  757. */
  758. if (!new_len)
  759. return ret;
  760. if (mmap_write_lock_killable(current->mm))
  761. return -EINTR;
  762. if (flags & (MREMAP_FIXED | MREMAP_DONTUNMAP)) {
  763. ret = mremap_to(addr, old_len, new_addr, new_len,
  764. &locked, flags, &uf, &uf_unmap_early,
  765. &uf_unmap);
  766. goto out;
  767. }
  768. /*
  769. * Always allow a shrinking remap: that just unmaps
  770. * the unnecessary pages..
  771. * __do_munmap does all the needed commit accounting, and
  772. * downgrades mmap_lock to read if so directed.
  773. */
  774. if (old_len >= new_len) {
  775. int retval;
  776. retval = __do_munmap(mm, addr+new_len, old_len - new_len,
  777. &uf_unmap, true);
  778. if (retval < 0 && old_len != new_len) {
  779. ret = retval;
  780. goto out;
  781. /* Returning 1 indicates mmap_lock is downgraded to read. */
  782. } else if (retval == 1)
  783. downgraded = true;
  784. ret = addr;
  785. goto out;
  786. }
  787. /*
  788. * Ok, we need to grow..
  789. */
  790. vma = vma_to_resize(addr, old_len, new_len, flags, &charged);
  791. if (IS_ERR(vma)) {
  792. ret = PTR_ERR(vma);
  793. goto out;
  794. }
  795. /* old_len exactly to the end of the area..
  796. */
  797. if (old_len == vma->vm_end - addr) {
  798. /* can we just expand the current mapping? */
  799. if (vma_expandable(vma, new_len - old_len)) {
  800. int pages = (new_len - old_len) >> PAGE_SHIFT;
  801. if (vma_adjust(vma, vma->vm_start, addr + new_len,
  802. vma->vm_pgoff, NULL)) {
  803. ret = -ENOMEM;
  804. goto out;
  805. }
  806. vm_stat_account(mm, vma->vm_flags, pages);
  807. if (vma->vm_flags & VM_LOCKED) {
  808. mm->locked_vm += pages;
  809. locked = true;
  810. new_addr = addr;
  811. }
  812. ret = addr;
  813. goto out;
  814. }
  815. }
  816. /*
  817. * We weren't able to just expand or shrink the area,
  818. * we need to create a new one and move it..
  819. */
  820. ret = -ENOMEM;
  821. if (flags & MREMAP_MAYMOVE) {
  822. unsigned long map_flags = 0;
  823. if (vma->vm_flags & VM_MAYSHARE)
  824. map_flags |= MAP_SHARED;
  825. new_addr = get_unmapped_area(vma->vm_file, 0, new_len,
  826. vma->vm_pgoff +
  827. ((addr - vma->vm_start) >> PAGE_SHIFT),
  828. map_flags);
  829. if (IS_ERR_VALUE(new_addr)) {
  830. ret = new_addr;
  831. goto out;
  832. }
  833. ret = move_vma(vma, addr, old_len, new_len, new_addr,
  834. &locked, flags, &uf, &uf_unmap);
  835. }
  836. out:
  837. if (offset_in_page(ret)) {
  838. vm_unacct_memory(charged);
  839. locked = false;
  840. }
  841. if (downgraded)
  842. mmap_read_unlock(current->mm);
  843. else
  844. mmap_write_unlock(current->mm);
  845. if (locked && new_len > old_len)
  846. mm_populate(new_addr + old_len, new_len - old_len);
  847. userfaultfd_unmap_complete(mm, &uf_unmap_early);
  848. mremap_userfaultfd_complete(&uf, addr, ret, old_len);
  849. userfaultfd_unmap_complete(mm, &uf_unmap);
  850. return ret;
  851. }