userfaultfd.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * mm/userfaultfd.c
  4. *
  5. * Copyright (C) 2015 Red Hat, Inc.
  6. */
  7. #include <linux/mm.h>
  8. #include <linux/sched/signal.h>
  9. #include <linux/pagemap.h>
  10. #include <linux/rmap.h>
  11. #include <linux/swap.h>
  12. #include <linux/swapops.h>
  13. #include <linux/userfaultfd_k.h>
  14. #include <linux/mmu_notifier.h>
  15. #include <linux/hugetlb.h>
  16. #include <linux/shmem_fs.h>
  17. #include <asm/tlbflush.h>
  18. #include "internal.h"
  19. static __always_inline
  20. struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm,
  21. unsigned long dst_start,
  22. unsigned long len)
  23. {
  24. /*
  25. * Make sure that the dst range is both valid and fully within a
  26. * single existing vma.
  27. */
  28. struct vm_area_struct *dst_vma;
  29. dst_vma = find_vma(dst_mm, dst_start);
  30. if (!dst_vma)
  31. return NULL;
  32. if (dst_start < dst_vma->vm_start ||
  33. dst_start + len > dst_vma->vm_end)
  34. return NULL;
  35. /*
  36. * Check the vma is registered in uffd, this is required to
  37. * enforce the VM_MAYWRITE check done at uffd registration
  38. * time.
  39. */
  40. if (!dst_vma->vm_userfaultfd_ctx.ctx)
  41. return NULL;
  42. return dst_vma;
  43. }
  44. /*
  45. * Install PTEs, to map dst_addr (within dst_vma) to page.
  46. *
  47. * This function handles both MCOPY_ATOMIC_NORMAL and _CONTINUE for both shmem
  48. * and anon, and for both shared and private VMAs.
  49. */
  50. int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
  51. struct vm_area_struct *dst_vma,
  52. unsigned long dst_addr, struct page *page,
  53. bool newly_allocated, bool wp_copy)
  54. {
  55. int ret;
  56. pte_t _dst_pte, *dst_pte;
  57. bool writable = dst_vma->vm_flags & VM_WRITE;
  58. bool vm_shared = dst_vma->vm_flags & VM_SHARED;
  59. bool page_in_cache = page->mapping;
  60. spinlock_t *ptl;
  61. struct inode *inode;
  62. pgoff_t offset, max_off;
  63. _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
  64. if (page_in_cache && !vm_shared)
  65. writable = false;
  66. if (writable || !page_in_cache)
  67. _dst_pte = pte_mkdirty(_dst_pte);
  68. if (writable) {
  69. if (wp_copy)
  70. _dst_pte = pte_mkuffd_wp(_dst_pte);
  71. else
  72. _dst_pte = pte_mkwrite(_dst_pte);
  73. }
  74. dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
  75. if (vma_is_shmem(dst_vma)) {
  76. /* serialize against truncate with the page table lock */
  77. inode = dst_vma->vm_file->f_inode;
  78. offset = linear_page_index(dst_vma, dst_addr);
  79. max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
  80. ret = -EFAULT;
  81. if (unlikely(offset >= max_off))
  82. goto out_unlock;
  83. }
  84. ret = -EEXIST;
  85. if (!pte_none(*dst_pte))
  86. goto out_unlock;
  87. if (page_in_cache)
  88. page_add_file_rmap(page, false);
  89. else
  90. page_add_new_anon_rmap(page, dst_vma, dst_addr, false);
  91. /*
  92. * Must happen after rmap, as mm_counter() checks mapping (via
  93. * PageAnon()), which is set by __page_set_anon_rmap().
  94. */
  95. inc_mm_counter(dst_mm, mm_counter(page));
  96. if (newly_allocated)
  97. lru_cache_add_inactive_or_unevictable(page, dst_vma);
  98. set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
  99. /* No need to invalidate - it was non-present before */
  100. update_mmu_cache(dst_vma, dst_addr, dst_pte);
  101. ret = 0;
  102. out_unlock:
  103. pte_unmap_unlock(dst_pte, ptl);
  104. return ret;
  105. }
  106. static int mcopy_atomic_pte(struct mm_struct *dst_mm,
  107. pmd_t *dst_pmd,
  108. struct vm_area_struct *dst_vma,
  109. unsigned long dst_addr,
  110. unsigned long src_addr,
  111. struct page **pagep,
  112. bool wp_copy)
  113. {
  114. void *page_kaddr;
  115. int ret;
  116. struct page *page;
  117. if (!*pagep) {
  118. ret = -ENOMEM;
  119. page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, dst_vma, dst_addr);
  120. if (!page)
  121. goto out;
  122. page_kaddr = kmap_atomic(page);
  123. ret = copy_from_user(page_kaddr,
  124. (const void __user *) src_addr,
  125. PAGE_SIZE);
  126. kunmap_atomic(page_kaddr);
  127. /* fallback to copy_from_user outside mmap_lock */
  128. if (unlikely(ret)) {
  129. ret = -ENOENT;
  130. *pagep = page;
  131. /* don't free the page */
  132. goto out;
  133. }
  134. } else {
  135. page = *pagep;
  136. *pagep = NULL;
  137. }
  138. /*
  139. * The memory barrier inside __SetPageUptodate makes sure that
  140. * preceding stores to the page contents become visible before
  141. * the set_pte_at() write.
  142. */
  143. __SetPageUptodate(page);
  144. ret = -ENOMEM;
  145. if (mem_cgroup_charge(page, dst_mm, GFP_KERNEL))
  146. goto out_release;
  147. ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
  148. page, true, wp_copy);
  149. if (ret)
  150. goto out_release;
  151. out:
  152. return ret;
  153. out_release:
  154. put_page(page);
  155. goto out;
  156. }
  157. static int mfill_zeropage_pte(struct mm_struct *dst_mm,
  158. pmd_t *dst_pmd,
  159. struct vm_area_struct *dst_vma,
  160. unsigned long dst_addr)
  161. {
  162. pte_t _dst_pte, *dst_pte;
  163. spinlock_t *ptl;
  164. int ret;
  165. pgoff_t offset, max_off;
  166. struct inode *inode;
  167. _dst_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr),
  168. dst_vma->vm_page_prot));
  169. dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
  170. if (dst_vma->vm_file) {
  171. /* the shmem MAP_PRIVATE case requires checking the i_size */
  172. inode = dst_vma->vm_file->f_inode;
  173. offset = linear_page_index(dst_vma, dst_addr);
  174. max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
  175. ret = -EFAULT;
  176. if (unlikely(offset >= max_off))
  177. goto out_unlock;
  178. }
  179. ret = -EEXIST;
  180. if (!pte_none(*dst_pte))
  181. goto out_unlock;
  182. set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
  183. /* No need to invalidate - it was non-present before */
  184. update_mmu_cache(dst_vma, dst_addr, dst_pte);
  185. ret = 0;
  186. out_unlock:
  187. pte_unmap_unlock(dst_pte, ptl);
  188. return ret;
  189. }
  190. /* Handles UFFDIO_CONTINUE for all shmem VMAs (shared or private). */
  191. static int mcontinue_atomic_pte(struct mm_struct *dst_mm,
  192. pmd_t *dst_pmd,
  193. struct vm_area_struct *dst_vma,
  194. unsigned long dst_addr,
  195. bool wp_copy)
  196. {
  197. struct inode *inode = file_inode(dst_vma->vm_file);
  198. pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
  199. struct page *page;
  200. int ret;
  201. ret = shmem_getpage(inode, pgoff, &page, SGP_READ);
  202. if (ret)
  203. goto out;
  204. if (!page) {
  205. ret = -EFAULT;
  206. goto out;
  207. }
  208. ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
  209. page, false, wp_copy);
  210. if (ret)
  211. goto out_release;
  212. unlock_page(page);
  213. ret = 0;
  214. out:
  215. return ret;
  216. out_release:
  217. unlock_page(page);
  218. put_page(page);
  219. goto out;
  220. }
  221. static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address)
  222. {
  223. pgd_t *pgd;
  224. p4d_t *p4d;
  225. pud_t *pud;
  226. pgd = pgd_offset(mm, address);
  227. p4d = p4d_alloc(mm, pgd, address);
  228. if (!p4d)
  229. return NULL;
  230. pud = pud_alloc(mm, p4d, address);
  231. if (!pud)
  232. return NULL;
  233. /*
  234. * Note that we didn't run this because the pmd was
  235. * missing, the *pmd may be already established and in
  236. * turn it may also be a trans_huge_pmd.
  237. */
  238. return pmd_alloc(mm, pud, address);
  239. }
  240. #ifdef CONFIG_HUGETLB_PAGE
  241. /*
  242. * __mcopy_atomic processing for HUGETLB vmas. Note that this routine is
  243. * called with mmap_lock held, it will release mmap_lock before returning.
  244. */
  245. static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
  246. struct vm_area_struct *dst_vma,
  247. unsigned long dst_start,
  248. unsigned long src_start,
  249. unsigned long len,
  250. enum mcopy_atomic_mode mode)
  251. {
  252. int vm_alloc_shared = dst_vma->vm_flags & VM_SHARED;
  253. int vm_shared = dst_vma->vm_flags & VM_SHARED;
  254. ssize_t err;
  255. pte_t *dst_pte;
  256. unsigned long src_addr, dst_addr;
  257. long copied;
  258. struct page *page;
  259. unsigned long vma_hpagesize;
  260. pgoff_t idx;
  261. u32 hash;
  262. struct address_space *mapping;
  263. /*
  264. * There is no default zero huge page for all huge page sizes as
  265. * supported by hugetlb. A PMD_SIZE huge pages may exist as used
  266. * by THP. Since we can not reliably insert a zero page, this
  267. * feature is not supported.
  268. */
  269. if (mode == MCOPY_ATOMIC_ZEROPAGE) {
  270. mmap_read_unlock(dst_mm);
  271. return -EINVAL;
  272. }
  273. src_addr = src_start;
  274. dst_addr = dst_start;
  275. copied = 0;
  276. page = NULL;
  277. vma_hpagesize = vma_kernel_pagesize(dst_vma);
  278. /*
  279. * Validate alignment based on huge page size
  280. */
  281. err = -EINVAL;
  282. if (dst_start & (vma_hpagesize - 1) || len & (vma_hpagesize - 1))
  283. goto out_unlock;
  284. retry:
  285. /*
  286. * On routine entry dst_vma is set. If we had to drop mmap_lock and
  287. * retry, dst_vma will be set to NULL and we must lookup again.
  288. */
  289. if (!dst_vma) {
  290. err = -ENOENT;
  291. dst_vma = find_dst_vma(dst_mm, dst_start, len);
  292. if (!dst_vma || !is_vm_hugetlb_page(dst_vma))
  293. goto out_unlock;
  294. err = -EINVAL;
  295. if (vma_hpagesize != vma_kernel_pagesize(dst_vma))
  296. goto out_unlock;
  297. vm_shared = dst_vma->vm_flags & VM_SHARED;
  298. }
  299. /*
  300. * If not shared, ensure the dst_vma has a anon_vma.
  301. */
  302. err = -ENOMEM;
  303. if (!vm_shared) {
  304. if (unlikely(anon_vma_prepare(dst_vma)))
  305. goto out_unlock;
  306. }
  307. while (src_addr < src_start + len) {
  308. BUG_ON(dst_addr >= dst_start + len);
  309. /*
  310. * Serialize via i_mmap_rwsem and hugetlb_fault_mutex.
  311. * i_mmap_rwsem ensures the dst_pte remains valid even
  312. * in the case of shared pmds. fault mutex prevents
  313. * races with other faulting threads.
  314. */
  315. mapping = dst_vma->vm_file->f_mapping;
  316. i_mmap_lock_read(mapping);
  317. idx = linear_page_index(dst_vma, dst_addr);
  318. hash = hugetlb_fault_mutex_hash(mapping, idx);
  319. mutex_lock(&hugetlb_fault_mutex_table[hash]);
  320. err = -ENOMEM;
  321. dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize);
  322. if (!dst_pte) {
  323. mutex_unlock(&hugetlb_fault_mutex_table[hash]);
  324. i_mmap_unlock_read(mapping);
  325. goto out_unlock;
  326. }
  327. if (mode != MCOPY_ATOMIC_CONTINUE &&
  328. !huge_pte_none(huge_ptep_get(dst_pte))) {
  329. err = -EEXIST;
  330. mutex_unlock(&hugetlb_fault_mutex_table[hash]);
  331. i_mmap_unlock_read(mapping);
  332. goto out_unlock;
  333. }
  334. err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma,
  335. dst_addr, src_addr, mode, &page);
  336. mutex_unlock(&hugetlb_fault_mutex_table[hash]);
  337. i_mmap_unlock_read(mapping);
  338. vm_alloc_shared = vm_shared;
  339. cond_resched();
  340. if (unlikely(err == -ENOENT)) {
  341. mmap_read_unlock(dst_mm);
  342. BUG_ON(!page);
  343. err = copy_huge_page_from_user(page,
  344. (const void __user *)src_addr,
  345. vma_hpagesize / PAGE_SIZE,
  346. true);
  347. if (unlikely(err)) {
  348. err = -EFAULT;
  349. goto out;
  350. }
  351. mmap_read_lock(dst_mm);
  352. dst_vma = NULL;
  353. goto retry;
  354. } else
  355. BUG_ON(page);
  356. if (!err) {
  357. dst_addr += vma_hpagesize;
  358. src_addr += vma_hpagesize;
  359. copied += vma_hpagesize;
  360. if (fatal_signal_pending(current))
  361. err = -EINTR;
  362. }
  363. if (err)
  364. break;
  365. }
  366. out_unlock:
  367. mmap_read_unlock(dst_mm);
  368. out:
  369. if (page) {
  370. /*
  371. * We encountered an error and are about to free a newly
  372. * allocated huge page.
  373. *
  374. * Reservation handling is very subtle, and is different for
  375. * private and shared mappings. See the routine
  376. * restore_reserve_on_error for details. Unfortunately, we
  377. * can not call restore_reserve_on_error now as it would
  378. * require holding mmap_lock.
  379. *
  380. * If a reservation for the page existed in the reservation
  381. * map of a private mapping, the map was modified to indicate
  382. * the reservation was consumed when the page was allocated.
  383. * We clear the PagePrivate flag now so that the global
  384. * reserve count will not be incremented in free_huge_page.
  385. * The reservation map will still indicate the reservation
  386. * was consumed and possibly prevent later page allocation.
  387. * This is better than leaking a global reservation. If no
  388. * reservation existed, it is still safe to clear PagePrivate
  389. * as no adjustments to reservation counts were made during
  390. * allocation.
  391. *
  392. * The reservation map for shared mappings indicates which
  393. * pages have reservations. When a huge page is allocated
  394. * for an address with a reservation, no change is made to
  395. * the reserve map. In this case PagePrivate will be set
  396. * to indicate that the global reservation count should be
  397. * incremented when the page is freed. This is the desired
  398. * behavior. However, when a huge page is allocated for an
  399. * address without a reservation a reservation entry is added
  400. * to the reservation map, and PagePrivate will not be set.
  401. * When the page is freed, the global reserve count will NOT
  402. * be incremented and it will appear as though we have leaked
  403. * reserved page. In this case, set PagePrivate so that the
  404. * global reserve count will be incremented to match the
  405. * reservation map entry which was created.
  406. *
  407. * Note that vm_alloc_shared is based on the flags of the vma
  408. * for which the page was originally allocated. dst_vma could
  409. * be different or NULL on error.
  410. */
  411. if (vm_alloc_shared)
  412. SetPagePrivate(page);
  413. else
  414. ClearPagePrivate(page);
  415. put_page(page);
  416. }
  417. BUG_ON(copied < 0);
  418. BUG_ON(err > 0);
  419. BUG_ON(!copied && !err);
  420. return copied ? copied : err;
  421. }
  422. #else /* !CONFIG_HUGETLB_PAGE */
  423. /* fail at build time if gcc attempts to use this */
  424. extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
  425. struct vm_area_struct *dst_vma,
  426. unsigned long dst_start,
  427. unsigned long src_start,
  428. unsigned long len,
  429. enum mcopy_atomic_mode mode);
  430. #endif /* CONFIG_HUGETLB_PAGE */
  431. static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm,
  432. pmd_t *dst_pmd,
  433. struct vm_area_struct *dst_vma,
  434. unsigned long dst_addr,
  435. unsigned long src_addr,
  436. struct page **page,
  437. enum mcopy_atomic_mode mode,
  438. bool wp_copy)
  439. {
  440. ssize_t err;
  441. if (mode == MCOPY_ATOMIC_CONTINUE) {
  442. return mcontinue_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
  443. wp_copy);
  444. }
  445. /*
  446. * The normal page fault path for a shmem will invoke the
  447. * fault, fill the hole in the file and COW it right away. The
  448. * result generates plain anonymous memory. So when we are
  449. * asked to fill an hole in a MAP_PRIVATE shmem mapping, we'll
  450. * generate anonymous memory directly without actually filling
  451. * the hole. For the MAP_PRIVATE case the robustness check
  452. * only happens in the pagetable (to verify it's still none)
  453. * and not in the radix tree.
  454. */
  455. if (!(dst_vma->vm_flags & VM_SHARED)) {
  456. if (mode == MCOPY_ATOMIC_NORMAL)
  457. err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma,
  458. dst_addr, src_addr, page,
  459. wp_copy);
  460. else
  461. err = mfill_zeropage_pte(dst_mm, dst_pmd,
  462. dst_vma, dst_addr);
  463. } else {
  464. VM_WARN_ON_ONCE(wp_copy);
  465. err = shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
  466. dst_addr, src_addr,
  467. mode != MCOPY_ATOMIC_NORMAL,
  468. page);
  469. }
  470. return err;
  471. }
  472. static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,
  473. unsigned long dst_start,
  474. unsigned long src_start,
  475. unsigned long len,
  476. enum mcopy_atomic_mode mcopy_mode,
  477. bool *mmap_changing,
  478. __u64 mode)
  479. {
  480. struct vm_area_struct *dst_vma;
  481. ssize_t err;
  482. pmd_t *dst_pmd;
  483. unsigned long src_addr, dst_addr;
  484. long copied;
  485. struct page *page;
  486. bool wp_copy;
  487. /*
  488. * Sanitize the command parameters:
  489. */
  490. BUG_ON(dst_start & ~PAGE_MASK);
  491. BUG_ON(len & ~PAGE_MASK);
  492. /* Does the address range wrap, or is the span zero-sized? */
  493. BUG_ON(src_start + len <= src_start);
  494. BUG_ON(dst_start + len <= dst_start);
  495. src_addr = src_start;
  496. dst_addr = dst_start;
  497. copied = 0;
  498. page = NULL;
  499. retry:
  500. mmap_read_lock(dst_mm);
  501. /*
  502. * If memory mappings are changing because of non-cooperative
  503. * operation (e.g. mremap) running in parallel, bail out and
  504. * request the user to retry later
  505. */
  506. err = -EAGAIN;
  507. if (mmap_changing && READ_ONCE(*mmap_changing))
  508. goto out_unlock;
  509. /*
  510. * Make sure the vma is not shared, that the dst range is
  511. * both valid and fully within a single existing vma.
  512. */
  513. err = -ENOENT;
  514. dst_vma = find_dst_vma(dst_mm, dst_start, len);
  515. if (!dst_vma)
  516. goto out_unlock;
  517. err = -EINVAL;
  518. /*
  519. * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but
  520. * it will overwrite vm_ops, so vma_is_anonymous must return false.
  521. */
  522. if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) &&
  523. dst_vma->vm_flags & VM_SHARED))
  524. goto out_unlock;
  525. /*
  526. * validate 'mode' now that we know the dst_vma: don't allow
  527. * a wrprotect copy if the userfaultfd didn't register as WP.
  528. */
  529. wp_copy = mode & UFFDIO_COPY_MODE_WP;
  530. if (wp_copy && !(dst_vma->vm_flags & VM_UFFD_WP))
  531. goto out_unlock;
  532. /*
  533. * If this is a HUGETLB vma, pass off to appropriate routine
  534. */
  535. if (is_vm_hugetlb_page(dst_vma))
  536. return __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start,
  537. src_start, len, mcopy_mode);
  538. if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma))
  539. goto out_unlock;
  540. if (!vma_is_shmem(dst_vma) && mcopy_mode == MCOPY_ATOMIC_CONTINUE)
  541. goto out_unlock;
  542. /*
  543. * Ensure the dst_vma has a anon_vma or this page
  544. * would get a NULL anon_vma when moved in the
  545. * dst_vma.
  546. */
  547. err = -ENOMEM;
  548. if (!(dst_vma->vm_flags & VM_SHARED) &&
  549. unlikely(anon_vma_prepare(dst_vma)))
  550. goto out_unlock;
  551. while (src_addr < src_start + len) {
  552. pmd_t dst_pmdval;
  553. BUG_ON(dst_addr >= dst_start + len);
  554. dst_pmd = mm_alloc_pmd(dst_mm, dst_addr);
  555. if (unlikely(!dst_pmd)) {
  556. err = -ENOMEM;
  557. break;
  558. }
  559. dst_pmdval = pmd_read_atomic(dst_pmd);
  560. /*
  561. * If the dst_pmd is mapped as THP don't
  562. * override it and just be strict.
  563. */
  564. if (unlikely(pmd_trans_huge(dst_pmdval))) {
  565. err = -EEXIST;
  566. break;
  567. }
  568. if (unlikely(pmd_none(dst_pmdval)) &&
  569. unlikely(__pte_alloc(dst_mm, dst_pmd))) {
  570. err = -ENOMEM;
  571. break;
  572. }
  573. /* If an huge pmd materialized from under us fail */
  574. if (unlikely(pmd_trans_huge(*dst_pmd))) {
  575. err = -EFAULT;
  576. break;
  577. }
  578. BUG_ON(pmd_none(*dst_pmd));
  579. BUG_ON(pmd_trans_huge(*dst_pmd));
  580. err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
  581. src_addr, &page, mcopy_mode, wp_copy);
  582. cond_resched();
  583. if (unlikely(err == -ENOENT)) {
  584. void *page_kaddr;
  585. mmap_read_unlock(dst_mm);
  586. BUG_ON(!page);
  587. page_kaddr = kmap(page);
  588. err = copy_from_user(page_kaddr,
  589. (const void __user *) src_addr,
  590. PAGE_SIZE);
  591. kunmap(page);
  592. if (unlikely(err)) {
  593. err = -EFAULT;
  594. goto out;
  595. }
  596. goto retry;
  597. } else
  598. BUG_ON(page);
  599. if (!err) {
  600. dst_addr += PAGE_SIZE;
  601. src_addr += PAGE_SIZE;
  602. copied += PAGE_SIZE;
  603. if (fatal_signal_pending(current))
  604. err = -EINTR;
  605. }
  606. if (err)
  607. break;
  608. }
  609. out_unlock:
  610. mmap_read_unlock(dst_mm);
  611. out:
  612. if (page)
  613. put_page(page);
  614. BUG_ON(copied < 0);
  615. BUG_ON(err > 0);
  616. BUG_ON(!copied && !err);
  617. return copied ? copied : err;
  618. }
  619. ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,
  620. unsigned long src_start, unsigned long len,
  621. bool *mmap_changing, __u64 mode)
  622. {
  623. return __mcopy_atomic(dst_mm, dst_start, src_start, len,
  624. MCOPY_ATOMIC_NORMAL, mmap_changing, mode);
  625. }
  626. ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start,
  627. unsigned long len, bool *mmap_changing)
  628. {
  629. return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_ZEROPAGE,
  630. mmap_changing, 0);
  631. }
  632. ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long start,
  633. unsigned long len, bool *mmap_changing)
  634. {
  635. return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_CONTINUE,
  636. mmap_changing, 0);
  637. }
  638. int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start,
  639. unsigned long len, bool enable_wp, bool *mmap_changing)
  640. {
  641. struct vm_area_struct *dst_vma;
  642. pgprot_t newprot;
  643. int err;
  644. /*
  645. * Sanitize the command parameters:
  646. */
  647. BUG_ON(start & ~PAGE_MASK);
  648. BUG_ON(len & ~PAGE_MASK);
  649. /* Does the address range wrap, or is the span zero-sized? */
  650. BUG_ON(start + len <= start);
  651. mmap_read_lock(dst_mm);
  652. /*
  653. * If memory mappings are changing because of non-cooperative
  654. * operation (e.g. mremap) running in parallel, bail out and
  655. * request the user to retry later
  656. */
  657. err = -EAGAIN;
  658. if (mmap_changing && READ_ONCE(*mmap_changing))
  659. goto out_unlock;
  660. err = -ENOENT;
  661. dst_vma = find_dst_vma(dst_mm, start, len);
  662. /*
  663. * Make sure the vma is not shared, that the dst range is
  664. * both valid and fully within a single existing vma.
  665. */
  666. if (!dst_vma || (dst_vma->vm_flags & VM_SHARED))
  667. goto out_unlock;
  668. if (!userfaultfd_wp(dst_vma))
  669. goto out_unlock;
  670. if (!vma_is_anonymous(dst_vma))
  671. goto out_unlock;
  672. if (enable_wp)
  673. newprot = vm_get_page_prot(dst_vma->vm_flags & ~(VM_WRITE));
  674. else
  675. newprot = vm_get_page_prot(dst_vma->vm_flags);
  676. change_protection(dst_vma, start, start + len, newprot,
  677. enable_wp ? MM_CP_UFFD_WP : MM_CP_UFFD_WP_RESOLVE);
  678. err = 0;
  679. out_unlock:
  680. mmap_read_unlock(dst_mm);
  681. return err;
  682. }