userfaultfd_k.h 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * include/linux/userfaultfd_k.h
  4. *
  5. * Copyright (C) 2015 Red Hat, Inc.
  6. *
  7. */
  8. #ifndef _LINUX_USERFAULTFD_K_H
  9. #define _LINUX_USERFAULTFD_K_H
  10. #ifdef CONFIG_USERFAULTFD
  11. #include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */
  12. #include <linux/fcntl.h>
  13. #include <linux/mm.h>
  14. #include <asm-generic/pgtable_uffd.h>
  15. /* The set of all possible UFFD-related VM flags. */
  16. #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR)
  17. /*
  18. * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining
  19. * new flags, since they might collide with O_* ones. We want
  20. * to re-use O_* flags that couldn't possibly have a meaning
  21. * from userfaultfd, in order to leave a free define-space for
  22. * shared O_* flags.
  23. */
  24. #define UFFD_CLOEXEC O_CLOEXEC
  25. #define UFFD_NONBLOCK O_NONBLOCK
  26. #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK)
  27. #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS)
  28. extern int sysctl_unprivileged_userfaultfd;
  29. extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason);
  30. /*
  31. * The mode of operation for __mcopy_atomic and its helpers.
  32. *
  33. * This is almost an implementation detail (mcopy_atomic below doesn't take this
  34. * as a parameter), but it's exposed here because memory-kind-specific
  35. * implementations (e.g. hugetlbfs) need to know the mode of operation.
  36. */
  37. enum mcopy_atomic_mode {
  38. /* A normal copy_from_user into the destination range. */
  39. MCOPY_ATOMIC_NORMAL,
  40. /* Don't copy; map the destination range to the zero page. */
  41. MCOPY_ATOMIC_ZEROPAGE,
  42. /* Just install pte(s) with the existing page(s) in the page cache. */
  43. MCOPY_ATOMIC_CONTINUE,
  44. };
  45. extern int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
  46. struct vm_area_struct *dst_vma,
  47. unsigned long dst_addr, struct page *page,
  48. bool newly_allocated, bool wp_copy);
  49. extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,
  50. unsigned long src_start, unsigned long len,
  51. bool *mmap_changing, __u64 mode);
  52. extern ssize_t mfill_zeropage(struct mm_struct *dst_mm,
  53. unsigned long dst_start,
  54. unsigned long len,
  55. bool *mmap_changing);
  56. extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start,
  57. unsigned long len, bool *mmap_changing);
  58. extern int mwriteprotect_range(struct mm_struct *dst_mm,
  59. unsigned long start, unsigned long len,
  60. bool enable_wp, bool *mmap_changing);
  61. /* mm helpers */
  62. static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma,
  63. struct vm_userfaultfd_ctx vm_ctx)
  64. {
  65. return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx;
  66. }
  67. /*
  68. * Never enable huge pmd sharing on some uffd registered vmas:
  69. *
  70. * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry.
  71. *
  72. * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for
  73. * VMAs which share huge pmds. (If you have two mappings to the same
  74. * underlying pages, and fault in the non-UFFD-registered one with a write,
  75. * with huge pmd sharing this would *also* setup the second UFFD-registered
  76. * mapping, and we'd not get minor faults.)
  77. */
  78. static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma)
  79. {
  80. return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR);
  81. }
  82. static inline bool userfaultfd_missing(struct vm_area_struct *vma)
  83. {
  84. return vma->vm_flags & VM_UFFD_MISSING;
  85. }
  86. static inline bool userfaultfd_wp(struct vm_area_struct *vma)
  87. {
  88. return vma->vm_flags & VM_UFFD_WP;
  89. }
  90. static inline bool userfaultfd_minor(struct vm_area_struct *vma)
  91. {
  92. return vma->vm_flags & VM_UFFD_MINOR;
  93. }
  94. static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
  95. pte_t pte)
  96. {
  97. return userfaultfd_wp(vma) && pte_uffd_wp(pte);
  98. }
  99. static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma,
  100. pmd_t pmd)
  101. {
  102. return userfaultfd_wp(vma) && pmd_uffd_wp(pmd);
  103. }
  104. static inline bool userfaultfd_armed(struct vm_area_struct *vma)
  105. {
  106. return vma->vm_flags & __VM_UFFD_FLAGS;
  107. }
  108. extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *);
  109. extern void dup_userfaultfd_complete(struct list_head *);
  110. extern void mremap_userfaultfd_prep(struct vm_area_struct *,
  111. struct vm_userfaultfd_ctx *);
  112. extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *,
  113. unsigned long from, unsigned long to,
  114. unsigned long len);
  115. extern bool userfaultfd_remove(struct vm_area_struct *vma,
  116. unsigned long start,
  117. unsigned long end);
  118. extern int userfaultfd_unmap_prep(struct vm_area_struct *vma,
  119. unsigned long start, unsigned long end,
  120. struct list_head *uf);
  121. extern void userfaultfd_unmap_complete(struct mm_struct *mm,
  122. struct list_head *uf);
  123. #else /* CONFIG_USERFAULTFD */
  124. /* mm helpers */
  125. static inline vm_fault_t handle_userfault(struct vm_fault *vmf,
  126. unsigned long reason)
  127. {
  128. return VM_FAULT_SIGBUS;
  129. }
  130. static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma,
  131. struct vm_userfaultfd_ctx vm_ctx)
  132. {
  133. return true;
  134. }
  135. static inline bool userfaultfd_missing(struct vm_area_struct *vma)
  136. {
  137. return false;
  138. }
  139. static inline bool userfaultfd_wp(struct vm_area_struct *vma)
  140. {
  141. return false;
  142. }
  143. static inline bool userfaultfd_minor(struct vm_area_struct *vma)
  144. {
  145. return false;
  146. }
  147. static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
  148. pte_t pte)
  149. {
  150. return false;
  151. }
  152. static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma,
  153. pmd_t pmd)
  154. {
  155. return false;
  156. }
  157. static inline bool userfaultfd_armed(struct vm_area_struct *vma)
  158. {
  159. return false;
  160. }
  161. static inline int dup_userfaultfd(struct vm_area_struct *vma,
  162. struct list_head *l)
  163. {
  164. return 0;
  165. }
  166. static inline void dup_userfaultfd_complete(struct list_head *l)
  167. {
  168. }
  169. static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma,
  170. struct vm_userfaultfd_ctx *ctx)
  171. {
  172. }
  173. static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx,
  174. unsigned long from,
  175. unsigned long to,
  176. unsigned long len)
  177. {
  178. }
  179. static inline bool userfaultfd_remove(struct vm_area_struct *vma,
  180. unsigned long start,
  181. unsigned long end)
  182. {
  183. return true;
  184. }
  185. static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma,
  186. unsigned long start, unsigned long end,
  187. struct list_head *uf)
  188. {
  189. return 0;
  190. }
  191. static inline void userfaultfd_unmap_complete(struct mm_struct *mm,
  192. struct list_head *uf)
  193. {
  194. }
  195. #endif /* CONFIG_USERFAULTFD */
  196. #endif /* _LINUX_USERFAULTFD_K_H */