userfaultfd.c 55 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * fs/userfaultfd.c
  4. *
  5. * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org>
  6. * Copyright (C) 2008-2009 Red Hat, Inc.
  7. * Copyright (C) 2015 Red Hat, Inc.
  8. *
  9. * Some part derived from fs/eventfd.c (anon inode setup) and
  10. * mm/ksm.c (mm hashing).
  11. */
  12. #include <linux/list.h>
  13. #include <linux/hashtable.h>
  14. #include <linux/sched/signal.h>
  15. #include <linux/sched/mm.h>
  16. #include <linux/mm.h>
  17. #include <linux/mmu_notifier.h>
  18. #include <linux/poll.h>
  19. #include <linux/slab.h>
  20. #include <linux/seq_file.h>
  21. #include <linux/file.h>
  22. #include <linux/bug.h>
  23. #include <linux/anon_inodes.h>
  24. #include <linux/syscalls.h>
  25. #include <linux/userfaultfd_k.h>
  26. #include <linux/mempolicy.h>
  27. #include <linux/ioctl.h>
  28. #include <linux/security.h>
  29. #include <linux/hugetlb.h>
  30. int sysctl_unprivileged_userfaultfd __read_mostly;
  31. static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly;
  32. /*
  33. * Start with fault_pending_wqh and fault_wqh so they're more likely
  34. * to be in the same cacheline.
  35. *
  36. * Locking order:
  37. * fd_wqh.lock
  38. * fault_pending_wqh.lock
  39. * fault_wqh.lock
  40. * event_wqh.lock
  41. *
  42. * To avoid deadlocks, IRQs must be disabled when taking any of the above locks,
  43. * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's
  44. * also taken in IRQ context.
  45. */
  46. struct userfaultfd_ctx {
  47. /* waitqueue head for the pending (i.e. not read) userfaults */
  48. wait_queue_head_t fault_pending_wqh;
  49. /* waitqueue head for the userfaults */
  50. wait_queue_head_t fault_wqh;
  51. /* waitqueue head for the pseudo fd to wakeup poll/read */
  52. wait_queue_head_t fd_wqh;
  53. /* waitqueue head for events */
  54. wait_queue_head_t event_wqh;
  55. /* a refile sequence protected by fault_pending_wqh lock */
  56. seqcount_spinlock_t refile_seq;
  57. /* pseudo fd refcounting */
  58. refcount_t refcount;
  59. /* userfaultfd syscall flags */
  60. unsigned int flags;
  61. /* features requested from the userspace */
  62. unsigned int features;
  63. /* released */
  64. bool released;
  65. /* memory mappings are changing because of non-cooperative event */
  66. bool mmap_changing;
  67. /* mm with one ore more vmas attached to this userfaultfd_ctx */
  68. struct mm_struct *mm;
  69. };
  70. struct userfaultfd_fork_ctx {
  71. struct userfaultfd_ctx *orig;
  72. struct userfaultfd_ctx *new;
  73. struct list_head list;
  74. };
  75. struct userfaultfd_unmap_ctx {
  76. struct userfaultfd_ctx *ctx;
  77. unsigned long start;
  78. unsigned long end;
  79. struct list_head list;
  80. };
  81. struct userfaultfd_wait_queue {
  82. struct uffd_msg msg;
  83. wait_queue_entry_t wq;
  84. struct userfaultfd_ctx *ctx;
  85. bool waken;
  86. };
  87. struct userfaultfd_wake_range {
  88. unsigned long start;
  89. unsigned long len;
  90. };
  91. /* internal indication that UFFD_API ioctl was successfully executed */
  92. #define UFFD_FEATURE_INITIALIZED (1u << 31)
  93. static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx)
  94. {
  95. return ctx->features & UFFD_FEATURE_INITIALIZED;
  96. }
  97. static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
  98. int wake_flags, void *key)
  99. {
  100. struct userfaultfd_wake_range *range = key;
  101. int ret;
  102. struct userfaultfd_wait_queue *uwq;
  103. unsigned long start, len;
  104. uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
  105. ret = 0;
  106. /* len == 0 means wake all */
  107. start = range->start;
  108. len = range->len;
  109. if (len && (start > uwq->msg.arg.pagefault.address ||
  110. start + len <= uwq->msg.arg.pagefault.address))
  111. goto out;
  112. WRITE_ONCE(uwq->waken, true);
  113. /*
  114. * The Program-Order guarantees provided by the scheduler
  115. * ensure uwq->waken is visible before the task is woken.
  116. */
  117. ret = wake_up_state(wq->private, mode);
  118. if (ret) {
  119. /*
  120. * Wake only once, autoremove behavior.
  121. *
  122. * After the effect of list_del_init is visible to the other
  123. * CPUs, the waitqueue may disappear from under us, see the
  124. * !list_empty_careful() in handle_userfault().
  125. *
  126. * try_to_wake_up() has an implicit smp_mb(), and the
  127. * wq->private is read before calling the extern function
  128. * "wake_up_state" (which in turns calls try_to_wake_up).
  129. */
  130. list_del_init(&wq->entry);
  131. }
  132. out:
  133. return ret;
  134. }
  135. /**
  136. * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
  137. * context.
  138. * @ctx: [in] Pointer to the userfaultfd context.
  139. */
  140. static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
  141. {
  142. refcount_inc(&ctx->refcount);
  143. }
  144. /**
  145. * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
  146. * context.
  147. * @ctx: [in] Pointer to userfaultfd context.
  148. *
  149. * The userfaultfd context reference must have been previously acquired either
  150. * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
  151. */
  152. static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
  153. {
  154. if (refcount_dec_and_test(&ctx->refcount)) {
  155. VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock));
  156. VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));
  157. VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));
  158. VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));
  159. VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));
  160. VM_BUG_ON(waitqueue_active(&ctx->event_wqh));
  161. VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
  162. VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
  163. mmdrop(ctx->mm);
  164. kmem_cache_free(userfaultfd_ctx_cachep, ctx);
  165. }
  166. }
  167. static inline void msg_init(struct uffd_msg *msg)
  168. {
  169. BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
  170. /*
  171. * Must use memset to zero out the paddings or kernel data is
  172. * leaked to userland.
  173. */
  174. memset(msg, 0, sizeof(struct uffd_msg));
  175. }
  176. static inline struct uffd_msg userfault_msg(unsigned long address,
  177. unsigned int flags,
  178. unsigned long reason,
  179. unsigned int features)
  180. {
  181. struct uffd_msg msg;
  182. msg_init(&msg);
  183. msg.event = UFFD_EVENT_PAGEFAULT;
  184. msg.arg.pagefault.address = address;
  185. /*
  186. * These flags indicate why the userfault occurred:
  187. * - UFFD_PAGEFAULT_FLAG_WP indicates a write protect fault.
  188. * - UFFD_PAGEFAULT_FLAG_MINOR indicates a minor fault.
  189. * - Neither of these flags being set indicates a MISSING fault.
  190. *
  191. * Separately, UFFD_PAGEFAULT_FLAG_WRITE indicates it was a write
  192. * fault. Otherwise, it was a read fault.
  193. */
  194. if (flags & FAULT_FLAG_WRITE)
  195. msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
  196. if (reason & VM_UFFD_WP)
  197. msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
  198. if (reason & VM_UFFD_MINOR)
  199. msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR;
  200. if (features & UFFD_FEATURE_THREAD_ID)
  201. msg.arg.pagefault.feat.ptid = task_pid_vnr(current);
  202. return msg;
  203. }
  204. #ifdef CONFIG_HUGETLB_PAGE
  205. /*
  206. * Same functionality as userfaultfd_must_wait below with modifications for
  207. * hugepmd ranges.
  208. */
  209. static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
  210. struct vm_area_struct *vma,
  211. unsigned long address,
  212. unsigned long flags,
  213. unsigned long reason)
  214. {
  215. struct mm_struct *mm = ctx->mm;
  216. pte_t *ptep, pte;
  217. bool ret = true;
  218. mmap_assert_locked(mm);
  219. ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));
  220. if (!ptep)
  221. goto out;
  222. ret = false;
  223. pte = huge_ptep_get(ptep);
  224. /*
  225. * Lockless access: we're in a wait_event so it's ok if it
  226. * changes under us.
  227. */
  228. if (huge_pte_none(pte))
  229. ret = true;
  230. if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
  231. ret = true;
  232. out:
  233. return ret;
  234. }
  235. #else
  236. static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
  237. struct vm_area_struct *vma,
  238. unsigned long address,
  239. unsigned long flags,
  240. unsigned long reason)
  241. {
  242. return false; /* should never get here */
  243. }
  244. #endif /* CONFIG_HUGETLB_PAGE */
  245. /*
  246. * Verify the pagetables are still not ok after having reigstered into
  247. * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
  248. * userfault that has already been resolved, if userfaultfd_read and
  249. * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
  250. * threads.
  251. */
  252. static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
  253. unsigned long address,
  254. unsigned long flags,
  255. unsigned long reason)
  256. {
  257. struct mm_struct *mm = ctx->mm;
  258. pgd_t *pgd;
  259. p4d_t *p4d;
  260. pud_t *pud;
  261. pmd_t *pmd, _pmd;
  262. pte_t *pte;
  263. bool ret = true;
  264. mmap_assert_locked(mm);
  265. pgd = pgd_offset(mm, address);
  266. if (!pgd_present(*pgd))
  267. goto out;
  268. p4d = p4d_offset(pgd, address);
  269. if (!p4d_present(*p4d))
  270. goto out;
  271. pud = pud_offset(p4d, address);
  272. if (!pud_present(*pud))
  273. goto out;
  274. pmd = pmd_offset(pud, address);
  275. /*
  276. * READ_ONCE must function as a barrier with narrower scope
  277. * and it must be equivalent to:
  278. * _pmd = *pmd; barrier();
  279. *
  280. * This is to deal with the instability (as in
  281. * pmd_trans_unstable) of the pmd.
  282. */
  283. _pmd = READ_ONCE(*pmd);
  284. if (pmd_none(_pmd))
  285. goto out;
  286. ret = false;
  287. if (!pmd_present(_pmd))
  288. goto out;
  289. if (pmd_trans_huge(_pmd)) {
  290. if (!pmd_write(_pmd) && (reason & VM_UFFD_WP))
  291. ret = true;
  292. goto out;
  293. }
  294. /*
  295. * the pmd is stable (as in !pmd_trans_unstable) so we can re-read it
  296. * and use the standard pte_offset_map() instead of parsing _pmd.
  297. */
  298. pte = pte_offset_map(pmd, address);
  299. /*
  300. * Lockless access: we're in a wait_event so it's ok if it
  301. * changes under us.
  302. */
  303. if (pte_none(*pte))
  304. ret = true;
  305. if (!pte_write(*pte) && (reason & VM_UFFD_WP))
  306. ret = true;
  307. pte_unmap(pte);
  308. out:
  309. return ret;
  310. }
  311. static inline long userfaultfd_get_blocking_state(unsigned int flags)
  312. {
  313. if (flags & FAULT_FLAG_INTERRUPTIBLE)
  314. return TASK_INTERRUPTIBLE;
  315. if (flags & FAULT_FLAG_KILLABLE)
  316. return TASK_KILLABLE;
  317. return TASK_UNINTERRUPTIBLE;
  318. }
  319. /*
  320. * The locking rules involved in returning VM_FAULT_RETRY depending on
  321. * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
  322. * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
  323. * recommendation in __lock_page_or_retry is not an understatement.
  324. *
  325. * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released
  326. * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
  327. * not set.
  328. *
  329. * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
  330. * set, VM_FAULT_RETRY can still be returned if and only if there are
  331. * fatal_signal_pending()s, and the mmap_lock must be released before
  332. * returning it.
  333. */
  334. vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
  335. {
  336. struct mm_struct *mm = vmf->vma->vm_mm;
  337. struct userfaultfd_ctx *ctx;
  338. struct userfaultfd_wait_queue uwq;
  339. vm_fault_t ret = VM_FAULT_SIGBUS;
  340. bool must_wait;
  341. long blocking_state;
  342. /*
  343. * We don't do userfault handling for the final child pid update.
  344. *
  345. * We also don't do userfault handling during
  346. * coredumping. hugetlbfs has the special
  347. * follow_hugetlb_page() to skip missing pages in the
  348. * FOLL_DUMP case, anon memory also checks for FOLL_DUMP with
  349. * the no_page_table() helper in follow_page_mask(), but the
  350. * shmem_vm_ops->fault method is invoked even during
  351. * coredumping without mmap_lock and it ends up here.
  352. */
  353. if (current->flags & (PF_EXITING|PF_DUMPCORE))
  354. goto out;
  355. /*
  356. * Coredumping runs without mmap_lock so we can only check that
  357. * the mmap_lock is held, if PF_DUMPCORE was not set.
  358. */
  359. mmap_assert_locked(mm);
  360. ctx = vmf->vma->vm_userfaultfd_ctx.ctx;
  361. if (!ctx)
  362. goto out;
  363. BUG_ON(ctx->mm != mm);
  364. /* Any unrecognized flag is a bug. */
  365. VM_BUG_ON(reason & ~__VM_UFFD_FLAGS);
  366. /* 0 or > 1 flags set is a bug; we expect exactly 1. */
  367. VM_BUG_ON(!reason || (reason & (reason - 1)));
  368. if (ctx->features & UFFD_FEATURE_SIGBUS)
  369. goto out;
  370. if ((vmf->flags & FAULT_FLAG_USER) == 0 &&
  371. ctx->flags & UFFD_USER_MODE_ONLY) {
  372. printk_once(KERN_WARNING "uffd: Set unprivileged_userfaultfd "
  373. "sysctl knob to 1 if kernel faults must be handled "
  374. "without obtaining CAP_SYS_PTRACE capability\n");
  375. goto out;
  376. }
  377. /*
  378. * If it's already released don't get it. This avoids to loop
  379. * in __get_user_pages if userfaultfd_release waits on the
  380. * caller of handle_userfault to release the mmap_lock.
  381. */
  382. if (unlikely(READ_ONCE(ctx->released))) {
  383. /*
  384. * Don't return VM_FAULT_SIGBUS in this case, so a non
  385. * cooperative manager can close the uffd after the
  386. * last UFFDIO_COPY, without risking to trigger an
  387. * involuntary SIGBUS if the process was starting the
  388. * userfaultfd while the userfaultfd was still armed
  389. * (but after the last UFFDIO_COPY). If the uffd
  390. * wasn't already closed when the userfault reached
  391. * this point, that would normally be solved by
  392. * userfaultfd_must_wait returning 'false'.
  393. *
  394. * If we were to return VM_FAULT_SIGBUS here, the non
  395. * cooperative manager would be instead forced to
  396. * always call UFFDIO_UNREGISTER before it can safely
  397. * close the uffd.
  398. */
  399. ret = VM_FAULT_NOPAGE;
  400. goto out;
  401. }
  402. /*
  403. * Check that we can return VM_FAULT_RETRY.
  404. *
  405. * NOTE: it should become possible to return VM_FAULT_RETRY
  406. * even if FAULT_FLAG_TRIED is set without leading to gup()
  407. * -EBUSY failures, if the userfaultfd is to be extended for
  408. * VM_UFFD_WP tracking and we intend to arm the userfault
  409. * without first stopping userland access to the memory. For
  410. * VM_UFFD_MISSING userfaults this is enough for now.
  411. */
  412. if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
  413. /*
  414. * Validate the invariant that nowait must allow retry
  415. * to be sure not to return SIGBUS erroneously on
  416. * nowait invocations.
  417. */
  418. BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
  419. #ifdef CONFIG_DEBUG_VM
  420. if (printk_ratelimit()) {
  421. printk(KERN_WARNING
  422. "FAULT_FLAG_ALLOW_RETRY missing %x\n",
  423. vmf->flags);
  424. dump_stack();
  425. }
  426. #endif
  427. goto out;
  428. }
  429. /*
  430. * Handle nowait, not much to do other than tell it to retry
  431. * and wait.
  432. */
  433. ret = VM_FAULT_RETRY;
  434. if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
  435. goto out;
  436. /* take the reference before dropping the mmap_lock */
  437. userfaultfd_ctx_get(ctx);
  438. init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
  439. uwq.wq.private = current;
  440. uwq.msg = userfault_msg(vmf->address, vmf->flags, reason,
  441. ctx->features);
  442. uwq.ctx = ctx;
  443. uwq.waken = false;
  444. blocking_state = userfaultfd_get_blocking_state(vmf->flags);
  445. spin_lock_irq(&ctx->fault_pending_wqh.lock);
  446. /*
  447. * After the __add_wait_queue the uwq is visible to userland
  448. * through poll/read().
  449. */
  450. __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
  451. /*
  452. * The smp_mb() after __set_current_state prevents the reads
  453. * following the spin_unlock to happen before the list_add in
  454. * __add_wait_queue.
  455. */
  456. set_current_state(blocking_state);
  457. spin_unlock_irq(&ctx->fault_pending_wqh.lock);
  458. if (!is_vm_hugetlb_page(vmf->vma))
  459. must_wait = userfaultfd_must_wait(ctx, vmf->address, vmf->flags,
  460. reason);
  461. else
  462. must_wait = userfaultfd_huge_must_wait(ctx, vmf->vma,
  463. vmf->address,
  464. vmf->flags, reason);
  465. mmap_read_unlock(mm);
  466. if (likely(must_wait && !READ_ONCE(ctx->released))) {
  467. wake_up_poll(&ctx->fd_wqh, EPOLLIN);
  468. schedule();
  469. }
  470. __set_current_state(TASK_RUNNING);
  471. /*
  472. * Here we race with the list_del; list_add in
  473. * userfaultfd_ctx_read(), however because we don't ever run
  474. * list_del_init() to refile across the two lists, the prev
  475. * and next pointers will never point to self. list_add also
  476. * would never let any of the two pointers to point to
  477. * self. So list_empty_careful won't risk to see both pointers
  478. * pointing to self at any time during the list refile. The
  479. * only case where list_del_init() is called is the full
  480. * removal in the wake function and there we don't re-list_add
  481. * and it's fine not to block on the spinlock. The uwq on this
  482. * kernel stack can be released after the list_del_init.
  483. */
  484. if (!list_empty_careful(&uwq.wq.entry)) {
  485. spin_lock_irq(&ctx->fault_pending_wqh.lock);
  486. /*
  487. * No need of list_del_init(), the uwq on the stack
  488. * will be freed shortly anyway.
  489. */
  490. list_del(&uwq.wq.entry);
  491. spin_unlock_irq(&ctx->fault_pending_wqh.lock);
  492. }
  493. /*
  494. * ctx may go away after this if the userfault pseudo fd is
  495. * already released.
  496. */
  497. userfaultfd_ctx_put(ctx);
  498. out:
  499. return ret;
  500. }
  501. static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
  502. struct userfaultfd_wait_queue *ewq)
  503. {
  504. struct userfaultfd_ctx *release_new_ctx;
  505. if (WARN_ON_ONCE(current->flags & PF_EXITING))
  506. goto out;
  507. ewq->ctx = ctx;
  508. init_waitqueue_entry(&ewq->wq, current);
  509. release_new_ctx = NULL;
  510. spin_lock_irq(&ctx->event_wqh.lock);
  511. /*
  512. * After the __add_wait_queue the uwq is visible to userland
  513. * through poll/read().
  514. */
  515. __add_wait_queue(&ctx->event_wqh, &ewq->wq);
  516. for (;;) {
  517. set_current_state(TASK_KILLABLE);
  518. if (ewq->msg.event == 0)
  519. break;
  520. if (READ_ONCE(ctx->released) ||
  521. fatal_signal_pending(current)) {
  522. /*
  523. * &ewq->wq may be queued in fork_event, but
  524. * __remove_wait_queue ignores the head
  525. * parameter. It would be a problem if it
  526. * didn't.
  527. */
  528. __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
  529. if (ewq->msg.event == UFFD_EVENT_FORK) {
  530. struct userfaultfd_ctx *new;
  531. new = (struct userfaultfd_ctx *)
  532. (unsigned long)
  533. ewq->msg.arg.reserved.reserved1;
  534. release_new_ctx = new;
  535. }
  536. break;
  537. }
  538. spin_unlock_irq(&ctx->event_wqh.lock);
  539. wake_up_poll(&ctx->fd_wqh, EPOLLIN);
  540. schedule();
  541. spin_lock_irq(&ctx->event_wqh.lock);
  542. }
  543. __set_current_state(TASK_RUNNING);
  544. spin_unlock_irq(&ctx->event_wqh.lock);
  545. if (release_new_ctx) {
  546. struct vm_area_struct *vma;
  547. struct mm_struct *mm = release_new_ctx->mm;
  548. /* the various vma->vm_userfaultfd_ctx still points to it */
  549. mmap_write_lock(mm);
  550. for (vma = mm->mmap; vma; vma = vma->vm_next)
  551. if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
  552. vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
  553. vma->vm_flags &= ~__VM_UFFD_FLAGS;
  554. }
  555. mmap_write_unlock(mm);
  556. userfaultfd_ctx_put(release_new_ctx);
  557. }
  558. /*
  559. * ctx may go away after this if the userfault pseudo fd is
  560. * already released.
  561. */
  562. out:
  563. WRITE_ONCE(ctx->mmap_changing, false);
  564. userfaultfd_ctx_put(ctx);
  565. }
  566. static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
  567. struct userfaultfd_wait_queue *ewq)
  568. {
  569. ewq->msg.event = 0;
  570. wake_up_locked(&ctx->event_wqh);
  571. __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
  572. }
  573. int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
  574. {
  575. struct userfaultfd_ctx *ctx = NULL, *octx;
  576. struct userfaultfd_fork_ctx *fctx;
  577. octx = vma->vm_userfaultfd_ctx.ctx;
  578. if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) {
  579. vm_write_begin(vma);
  580. vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
  581. WRITE_ONCE(vma->vm_flags,
  582. vma->vm_flags & ~__VM_UFFD_FLAGS);
  583. vm_write_end(vma);
  584. return 0;
  585. }
  586. list_for_each_entry(fctx, fcs, list)
  587. if (fctx->orig == octx) {
  588. ctx = fctx->new;
  589. break;
  590. }
  591. if (!ctx) {
  592. fctx = kmalloc(sizeof(*fctx), GFP_KERNEL);
  593. if (!fctx)
  594. return -ENOMEM;
  595. ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
  596. if (!ctx) {
  597. kfree(fctx);
  598. return -ENOMEM;
  599. }
  600. refcount_set(&ctx->refcount, 1);
  601. ctx->flags = octx->flags;
  602. ctx->features = octx->features;
  603. ctx->released = false;
  604. ctx->mmap_changing = false;
  605. ctx->mm = vma->vm_mm;
  606. mmgrab(ctx->mm);
  607. userfaultfd_ctx_get(octx);
  608. WRITE_ONCE(octx->mmap_changing, true);
  609. fctx->orig = octx;
  610. fctx->new = ctx;
  611. list_add_tail(&fctx->list, fcs);
  612. }
  613. vma->vm_userfaultfd_ctx.ctx = ctx;
  614. return 0;
  615. }
  616. static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
  617. {
  618. struct userfaultfd_ctx *ctx = fctx->orig;
  619. struct userfaultfd_wait_queue ewq;
  620. msg_init(&ewq.msg);
  621. ewq.msg.event = UFFD_EVENT_FORK;
  622. ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
  623. userfaultfd_event_wait_completion(ctx, &ewq);
  624. }
  625. void dup_userfaultfd_complete(struct list_head *fcs)
  626. {
  627. struct userfaultfd_fork_ctx *fctx, *n;
  628. list_for_each_entry_safe(fctx, n, fcs, list) {
  629. dup_fctx(fctx);
  630. list_del(&fctx->list);
  631. kfree(fctx);
  632. }
  633. }
  634. void mremap_userfaultfd_prep(struct vm_area_struct *vma,
  635. struct vm_userfaultfd_ctx *vm_ctx)
  636. {
  637. struct userfaultfd_ctx *ctx;
  638. ctx = vma->vm_userfaultfd_ctx.ctx;
  639. if (!ctx)
  640. return;
  641. if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {
  642. vm_ctx->ctx = ctx;
  643. userfaultfd_ctx_get(ctx);
  644. WRITE_ONCE(ctx->mmap_changing, true);
  645. } else {
  646. /* Drop uffd context if remap feature not enabled */
  647. vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
  648. vma->vm_flags &= ~__VM_UFFD_FLAGS;
  649. }
  650. }
  651. void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,
  652. unsigned long from, unsigned long to,
  653. unsigned long len)
  654. {
  655. struct userfaultfd_ctx *ctx = vm_ctx->ctx;
  656. struct userfaultfd_wait_queue ewq;
  657. if (!ctx)
  658. return;
  659. if (to & ~PAGE_MASK) {
  660. userfaultfd_ctx_put(ctx);
  661. return;
  662. }
  663. msg_init(&ewq.msg);
  664. ewq.msg.event = UFFD_EVENT_REMAP;
  665. ewq.msg.arg.remap.from = from;
  666. ewq.msg.arg.remap.to = to;
  667. ewq.msg.arg.remap.len = len;
  668. userfaultfd_event_wait_completion(ctx, &ewq);
  669. }
  670. bool userfaultfd_remove(struct vm_area_struct *vma,
  671. unsigned long start, unsigned long end)
  672. {
  673. struct mm_struct *mm = vma->vm_mm;
  674. struct userfaultfd_ctx *ctx;
  675. struct userfaultfd_wait_queue ewq;
  676. ctx = vma->vm_userfaultfd_ctx.ctx;
  677. if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
  678. return true;
  679. userfaultfd_ctx_get(ctx);
  680. WRITE_ONCE(ctx->mmap_changing, true);
  681. mmap_read_unlock(mm);
  682. msg_init(&ewq.msg);
  683. ewq.msg.event = UFFD_EVENT_REMOVE;
  684. ewq.msg.arg.remove.start = start;
  685. ewq.msg.arg.remove.end = end;
  686. userfaultfd_event_wait_completion(ctx, &ewq);
  687. return false;
  688. }
  689. static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
  690. unsigned long start, unsigned long end)
  691. {
  692. struct userfaultfd_unmap_ctx *unmap_ctx;
  693. list_for_each_entry(unmap_ctx, unmaps, list)
  694. if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&
  695. unmap_ctx->end == end)
  696. return true;
  697. return false;
  698. }
  699. int userfaultfd_unmap_prep(struct vm_area_struct *vma,
  700. unsigned long start, unsigned long end,
  701. struct list_head *unmaps)
  702. {
  703. for ( ; vma && vma->vm_start < end; vma = vma->vm_next) {
  704. struct userfaultfd_unmap_ctx *unmap_ctx;
  705. struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
  706. if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||
  707. has_unmap_ctx(ctx, unmaps, start, end))
  708. continue;
  709. unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL);
  710. if (!unmap_ctx)
  711. return -ENOMEM;
  712. userfaultfd_ctx_get(ctx);
  713. WRITE_ONCE(ctx->mmap_changing, true);
  714. unmap_ctx->ctx = ctx;
  715. unmap_ctx->start = start;
  716. unmap_ctx->end = end;
  717. list_add_tail(&unmap_ctx->list, unmaps);
  718. }
  719. return 0;
  720. }
  721. void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
  722. {
  723. struct userfaultfd_unmap_ctx *ctx, *n;
  724. struct userfaultfd_wait_queue ewq;
  725. list_for_each_entry_safe(ctx, n, uf, list) {
  726. msg_init(&ewq.msg);
  727. ewq.msg.event = UFFD_EVENT_UNMAP;
  728. ewq.msg.arg.remove.start = ctx->start;
  729. ewq.msg.arg.remove.end = ctx->end;
  730. userfaultfd_event_wait_completion(ctx->ctx, &ewq);
  731. list_del(&ctx->list);
  732. kfree(ctx);
  733. }
  734. }
  735. static int userfaultfd_release(struct inode *inode, struct file *file)
  736. {
  737. struct userfaultfd_ctx *ctx = file->private_data;
  738. struct mm_struct *mm = ctx->mm;
  739. struct vm_area_struct *vma, *prev;
  740. /* len == 0 means wake all */
  741. struct userfaultfd_wake_range range = { .len = 0, };
  742. unsigned long new_flags;
  743. WRITE_ONCE(ctx->released, true);
  744. if (!mmget_not_zero(mm))
  745. goto wakeup;
  746. /*
  747. * Flush page faults out of all CPUs. NOTE: all page faults
  748. * must be retried without returning VM_FAULT_SIGBUS if
  749. * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx
  750. * changes while handle_userfault released the mmap_lock. So
  751. * it's critical that released is set to true (above), before
  752. * taking the mmap_lock for writing.
  753. */
  754. mmap_write_lock(mm);
  755. prev = NULL;
  756. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  757. cond_resched();
  758. BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
  759. !!(vma->vm_flags & __VM_UFFD_FLAGS));
  760. if (vma->vm_userfaultfd_ctx.ctx != ctx) {
  761. prev = vma;
  762. continue;
  763. }
  764. new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
  765. prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
  766. new_flags, vma->anon_vma,
  767. vma->vm_file, vma->vm_pgoff,
  768. vma_policy(vma),
  769. NULL_VM_UFFD_CTX,
  770. vma_get_anon_name(vma));
  771. if (prev)
  772. vma = prev;
  773. else
  774. prev = vma;
  775. vm_write_begin(vma);
  776. WRITE_ONCE(vma->vm_flags, new_flags);
  777. vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
  778. vm_write_end(vma);
  779. }
  780. mmap_write_unlock(mm);
  781. mmput(mm);
  782. wakeup:
  783. /*
  784. * After no new page faults can wait on this fault_*wqh, flush
  785. * the last page faults that may have been already waiting on
  786. * the fault_*wqh.
  787. */
  788. spin_lock_irq(&ctx->fault_pending_wqh.lock);
  789. __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
  790. __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
  791. spin_unlock_irq(&ctx->fault_pending_wqh.lock);
  792. /* Flush pending events that may still wait on event_wqh */
  793. wake_up_all(&ctx->event_wqh);
  794. wake_up_poll(&ctx->fd_wqh, EPOLLHUP);
  795. userfaultfd_ctx_put(ctx);
  796. return 0;
  797. }
  798. /* fault_pending_wqh.lock must be hold by the caller */
  799. static inline struct userfaultfd_wait_queue *find_userfault_in(
  800. wait_queue_head_t *wqh)
  801. {
  802. wait_queue_entry_t *wq;
  803. struct userfaultfd_wait_queue *uwq;
  804. lockdep_assert_held(&wqh->lock);
  805. uwq = NULL;
  806. if (!waitqueue_active(wqh))
  807. goto out;
  808. /* walk in reverse to provide FIFO behavior to read userfaults */
  809. wq = list_last_entry(&wqh->head, typeof(*wq), entry);
  810. uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
  811. out:
  812. return uwq;
  813. }
  814. static inline struct userfaultfd_wait_queue *find_userfault(
  815. struct userfaultfd_ctx *ctx)
  816. {
  817. return find_userfault_in(&ctx->fault_pending_wqh);
  818. }
  819. static inline struct userfaultfd_wait_queue *find_userfault_evt(
  820. struct userfaultfd_ctx *ctx)
  821. {
  822. return find_userfault_in(&ctx->event_wqh);
  823. }
  824. static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)
  825. {
  826. struct userfaultfd_ctx *ctx = file->private_data;
  827. __poll_t ret;
  828. poll_wait(file, &ctx->fd_wqh, wait);
  829. if (!userfaultfd_is_initialized(ctx))
  830. return EPOLLERR;
  831. /*
  832. * poll() never guarantees that read won't block.
  833. * userfaults can be waken before they're read().
  834. */
  835. if (unlikely(!(file->f_flags & O_NONBLOCK)))
  836. return EPOLLERR;
  837. /*
  838. * lockless access to see if there are pending faults
  839. * __pollwait last action is the add_wait_queue but
  840. * the spin_unlock would allow the waitqueue_active to
  841. * pass above the actual list_add inside
  842. * add_wait_queue critical section. So use a full
  843. * memory barrier to serialize the list_add write of
  844. * add_wait_queue() with the waitqueue_active read
  845. * below.
  846. */
  847. ret = 0;
  848. smp_mb();
  849. if (waitqueue_active(&ctx->fault_pending_wqh))
  850. ret = EPOLLIN;
  851. else if (waitqueue_active(&ctx->event_wqh))
  852. ret = EPOLLIN;
  853. return ret;
  854. }
  855. static const struct file_operations userfaultfd_fops;
  856. static int resolve_userfault_fork(struct userfaultfd_ctx *new,
  857. struct inode *inode,
  858. struct uffd_msg *msg)
  859. {
  860. int fd;
  861. fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, new,
  862. O_RDWR | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode);
  863. if (fd < 0)
  864. return fd;
  865. msg->arg.reserved.reserved1 = 0;
  866. msg->arg.fork.ufd = fd;
  867. return 0;
  868. }
  869. static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
  870. struct uffd_msg *msg, struct inode *inode)
  871. {
  872. ssize_t ret;
  873. DECLARE_WAITQUEUE(wait, current);
  874. struct userfaultfd_wait_queue *uwq;
  875. /*
  876. * Handling fork event requires sleeping operations, so
  877. * we drop the event_wqh lock, then do these ops, then
  878. * lock it back and wake up the waiter. While the lock is
  879. * dropped the ewq may go away so we keep track of it
  880. * carefully.
  881. */
  882. LIST_HEAD(fork_event);
  883. struct userfaultfd_ctx *fork_nctx = NULL;
  884. /* always take the fd_wqh lock before the fault_pending_wqh lock */
  885. spin_lock_irq(&ctx->fd_wqh.lock);
  886. __add_wait_queue(&ctx->fd_wqh, &wait);
  887. for (;;) {
  888. set_current_state(TASK_INTERRUPTIBLE);
  889. spin_lock(&ctx->fault_pending_wqh.lock);
  890. uwq = find_userfault(ctx);
  891. if (uwq) {
  892. /*
  893. * Use a seqcount to repeat the lockless check
  894. * in wake_userfault() to avoid missing
  895. * wakeups because during the refile both
  896. * waitqueue could become empty if this is the
  897. * only userfault.
  898. */
  899. write_seqcount_begin(&ctx->refile_seq);
  900. /*
  901. * The fault_pending_wqh.lock prevents the uwq
  902. * to disappear from under us.
  903. *
  904. * Refile this userfault from
  905. * fault_pending_wqh to fault_wqh, it's not
  906. * pending anymore after we read it.
  907. *
  908. * Use list_del() by hand (as
  909. * userfaultfd_wake_function also uses
  910. * list_del_init() by hand) to be sure nobody
  911. * changes __remove_wait_queue() to use
  912. * list_del_init() in turn breaking the
  913. * !list_empty_careful() check in
  914. * handle_userfault(). The uwq->wq.head list
  915. * must never be empty at any time during the
  916. * refile, or the waitqueue could disappear
  917. * from under us. The "wait_queue_head_t"
  918. * parameter of __remove_wait_queue() is unused
  919. * anyway.
  920. */
  921. list_del(&uwq->wq.entry);
  922. add_wait_queue(&ctx->fault_wqh, &uwq->wq);
  923. write_seqcount_end(&ctx->refile_seq);
  924. /* careful to always initialize msg if ret == 0 */
  925. *msg = uwq->msg;
  926. spin_unlock(&ctx->fault_pending_wqh.lock);
  927. ret = 0;
  928. break;
  929. }
  930. spin_unlock(&ctx->fault_pending_wqh.lock);
  931. spin_lock(&ctx->event_wqh.lock);
  932. uwq = find_userfault_evt(ctx);
  933. if (uwq) {
  934. *msg = uwq->msg;
  935. if (uwq->msg.event == UFFD_EVENT_FORK) {
  936. fork_nctx = (struct userfaultfd_ctx *)
  937. (unsigned long)
  938. uwq->msg.arg.reserved.reserved1;
  939. list_move(&uwq->wq.entry, &fork_event);
  940. /*
  941. * fork_nctx can be freed as soon as
  942. * we drop the lock, unless we take a
  943. * reference on it.
  944. */
  945. userfaultfd_ctx_get(fork_nctx);
  946. spin_unlock(&ctx->event_wqh.lock);
  947. ret = 0;
  948. break;
  949. }
  950. userfaultfd_event_complete(ctx, uwq);
  951. spin_unlock(&ctx->event_wqh.lock);
  952. ret = 0;
  953. break;
  954. }
  955. spin_unlock(&ctx->event_wqh.lock);
  956. if (signal_pending(current)) {
  957. ret = -ERESTARTSYS;
  958. break;
  959. }
  960. if (no_wait) {
  961. ret = -EAGAIN;
  962. break;
  963. }
  964. spin_unlock_irq(&ctx->fd_wqh.lock);
  965. schedule();
  966. spin_lock_irq(&ctx->fd_wqh.lock);
  967. }
  968. __remove_wait_queue(&ctx->fd_wqh, &wait);
  969. __set_current_state(TASK_RUNNING);
  970. spin_unlock_irq(&ctx->fd_wqh.lock);
  971. if (!ret && msg->event == UFFD_EVENT_FORK) {
  972. ret = resolve_userfault_fork(fork_nctx, inode, msg);
  973. spin_lock_irq(&ctx->event_wqh.lock);
  974. if (!list_empty(&fork_event)) {
  975. /*
  976. * The fork thread didn't abort, so we can
  977. * drop the temporary refcount.
  978. */
  979. userfaultfd_ctx_put(fork_nctx);
  980. uwq = list_first_entry(&fork_event,
  981. typeof(*uwq),
  982. wq.entry);
  983. /*
  984. * If fork_event list wasn't empty and in turn
  985. * the event wasn't already released by fork
  986. * (the event is allocated on fork kernel
  987. * stack), put the event back to its place in
  988. * the event_wq. fork_event head will be freed
  989. * as soon as we return so the event cannot
  990. * stay queued there no matter the current
  991. * "ret" value.
  992. */
  993. list_del(&uwq->wq.entry);
  994. __add_wait_queue(&ctx->event_wqh, &uwq->wq);
  995. /*
  996. * Leave the event in the waitqueue and report
  997. * error to userland if we failed to resolve
  998. * the userfault fork.
  999. */
  1000. if (likely(!ret))
  1001. userfaultfd_event_complete(ctx, uwq);
  1002. } else {
  1003. /*
  1004. * Here the fork thread aborted and the
  1005. * refcount from the fork thread on fork_nctx
  1006. * has already been released. We still hold
  1007. * the reference we took before releasing the
  1008. * lock above. If resolve_userfault_fork
  1009. * failed we've to drop it because the
  1010. * fork_nctx has to be freed in such case. If
  1011. * it succeeded we'll hold it because the new
  1012. * uffd references it.
  1013. */
  1014. if (ret)
  1015. userfaultfd_ctx_put(fork_nctx);
  1016. }
  1017. spin_unlock_irq(&ctx->event_wqh.lock);
  1018. }
  1019. return ret;
  1020. }
  1021. static ssize_t userfaultfd_read(struct file *file, char __user *buf,
  1022. size_t count, loff_t *ppos)
  1023. {
  1024. struct userfaultfd_ctx *ctx = file->private_data;
  1025. ssize_t _ret, ret = 0;
  1026. struct uffd_msg msg;
  1027. int no_wait = file->f_flags & O_NONBLOCK;
  1028. struct inode *inode = file_inode(file);
  1029. if (!userfaultfd_is_initialized(ctx))
  1030. return -EINVAL;
  1031. for (;;) {
  1032. if (count < sizeof(msg))
  1033. return ret ? ret : -EINVAL;
  1034. _ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode);
  1035. if (_ret < 0)
  1036. return ret ? ret : _ret;
  1037. if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg)))
  1038. return ret ? ret : -EFAULT;
  1039. ret += sizeof(msg);
  1040. buf += sizeof(msg);
  1041. count -= sizeof(msg);
  1042. /*
  1043. * Allow to read more than one fault at time but only
  1044. * block if waiting for the very first one.
  1045. */
  1046. no_wait = O_NONBLOCK;
  1047. }
  1048. }
  1049. static void __wake_userfault(struct userfaultfd_ctx *ctx,
  1050. struct userfaultfd_wake_range *range)
  1051. {
  1052. spin_lock_irq(&ctx->fault_pending_wqh.lock);
  1053. /* wake all in the range and autoremove */
  1054. if (waitqueue_active(&ctx->fault_pending_wqh))
  1055. __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
  1056. range);
  1057. if (waitqueue_active(&ctx->fault_wqh))
  1058. __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);
  1059. spin_unlock_irq(&ctx->fault_pending_wqh.lock);
  1060. }
  1061. static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
  1062. struct userfaultfd_wake_range *range)
  1063. {
  1064. unsigned seq;
  1065. bool need_wakeup;
  1066. /*
  1067. * To be sure waitqueue_active() is not reordered by the CPU
  1068. * before the pagetable update, use an explicit SMP memory
  1069. * barrier here. PT lock release or mmap_read_unlock(mm) still
  1070. * have release semantics that can allow the
  1071. * waitqueue_active() to be reordered before the pte update.
  1072. */
  1073. smp_mb();
  1074. /*
  1075. * Use waitqueue_active because it's very frequent to
  1076. * change the address space atomically even if there are no
  1077. * userfaults yet. So we take the spinlock only when we're
  1078. * sure we've userfaults to wake.
  1079. */
  1080. do {
  1081. seq = read_seqcount_begin(&ctx->refile_seq);
  1082. need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
  1083. waitqueue_active(&ctx->fault_wqh);
  1084. cond_resched();
  1085. } while (read_seqcount_retry(&ctx->refile_seq, seq));
  1086. if (need_wakeup)
  1087. __wake_userfault(ctx, range);
  1088. }
  1089. static __always_inline int validate_range(struct mm_struct *mm,
  1090. __u64 start, __u64 len)
  1091. {
  1092. __u64 task_size = mm->task_size;
  1093. if (start & ~PAGE_MASK)
  1094. return -EINVAL;
  1095. if (len & ~PAGE_MASK)
  1096. return -EINVAL;
  1097. if (!len)
  1098. return -EINVAL;
  1099. if (start < mmap_min_addr)
  1100. return -EINVAL;
  1101. if (start >= task_size)
  1102. return -EINVAL;
  1103. if (len > task_size - start)
  1104. return -EINVAL;
  1105. return 0;
  1106. }
  1107. static inline bool vma_can_userfault(struct vm_area_struct *vma,
  1108. unsigned long vm_flags)
  1109. {
  1110. /* FIXME: add WP support to hugetlbfs and shmem */
  1111. if (vm_flags & VM_UFFD_WP) {
  1112. if (is_vm_hugetlb_page(vma) || vma_is_shmem(vma))
  1113. return false;
  1114. }
  1115. if (vm_flags & VM_UFFD_MINOR) {
  1116. if (!(is_vm_hugetlb_page(vma) || vma_is_shmem(vma)))
  1117. return false;
  1118. }
  1119. return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) ||
  1120. vma_is_shmem(vma);
  1121. }
  1122. static int userfaultfd_register(struct userfaultfd_ctx *ctx,
  1123. unsigned long arg)
  1124. {
  1125. struct mm_struct *mm = ctx->mm;
  1126. struct vm_area_struct *vma, *prev, *cur;
  1127. int ret;
  1128. struct uffdio_register uffdio_register;
  1129. struct uffdio_register __user *user_uffdio_register;
  1130. unsigned long vm_flags, new_flags;
  1131. bool found;
  1132. bool basic_ioctls;
  1133. unsigned long start, end, vma_end;
  1134. user_uffdio_register = (struct uffdio_register __user *) arg;
  1135. ret = -EFAULT;
  1136. if (copy_from_user(&uffdio_register, user_uffdio_register,
  1137. sizeof(uffdio_register)-sizeof(__u64)))
  1138. goto out;
  1139. ret = -EINVAL;
  1140. if (!uffdio_register.mode)
  1141. goto out;
  1142. if (uffdio_register.mode & ~UFFD_API_REGISTER_MODES)
  1143. goto out;
  1144. vm_flags = 0;
  1145. if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
  1146. vm_flags |= VM_UFFD_MISSING;
  1147. if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP)
  1148. vm_flags |= VM_UFFD_WP;
  1149. if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
  1150. #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
  1151. goto out;
  1152. #endif
  1153. vm_flags |= VM_UFFD_MINOR;
  1154. }
  1155. ret = validate_range(mm, uffdio_register.range.start,
  1156. uffdio_register.range.len);
  1157. if (ret)
  1158. goto out;
  1159. start = uffdio_register.range.start;
  1160. end = start + uffdio_register.range.len;
  1161. ret = -ENOMEM;
  1162. if (!mmget_not_zero(mm))
  1163. goto out;
  1164. mmap_write_lock(mm);
  1165. vma = find_vma_prev(mm, start, &prev);
  1166. if (!vma)
  1167. goto out_unlock;
  1168. /* check that there's at least one vma in the range */
  1169. ret = -EINVAL;
  1170. if (vma->vm_start >= end)
  1171. goto out_unlock;
  1172. /*
  1173. * If the first vma contains huge pages, make sure start address
  1174. * is aligned to huge page size.
  1175. */
  1176. if (is_vm_hugetlb_page(vma)) {
  1177. unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
  1178. if (start & (vma_hpagesize - 1))
  1179. goto out_unlock;
  1180. }
  1181. /*
  1182. * Search for not compatible vmas.
  1183. */
  1184. found = false;
  1185. basic_ioctls = false;
  1186. for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
  1187. cond_resched();
  1188. BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
  1189. !!(cur->vm_flags & __VM_UFFD_FLAGS));
  1190. /* check not compatible vmas */
  1191. ret = -EINVAL;
  1192. if (!vma_can_userfault(cur, vm_flags))
  1193. goto out_unlock;
  1194. /*
  1195. * UFFDIO_COPY will fill file holes even without
  1196. * PROT_WRITE. This check enforces that if this is a
  1197. * MAP_SHARED, the process has write permission to the backing
  1198. * file. If VM_MAYWRITE is set it also enforces that on a
  1199. * MAP_SHARED vma: there is no F_WRITE_SEAL and no further
  1200. * F_WRITE_SEAL can be taken until the vma is destroyed.
  1201. */
  1202. ret = -EPERM;
  1203. if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))
  1204. goto out_unlock;
  1205. /*
  1206. * If this vma contains ending address, and huge pages
  1207. * check alignment.
  1208. */
  1209. if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
  1210. end > cur->vm_start) {
  1211. unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
  1212. ret = -EINVAL;
  1213. if (end & (vma_hpagesize - 1))
  1214. goto out_unlock;
  1215. }
  1216. if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE))
  1217. goto out_unlock;
  1218. /*
  1219. * Check that this vma isn't already owned by a
  1220. * different userfaultfd. We can't allow more than one
  1221. * userfaultfd to own a single vma simultaneously or we
  1222. * wouldn't know which one to deliver the userfaults to.
  1223. */
  1224. ret = -EBUSY;
  1225. if (cur->vm_userfaultfd_ctx.ctx &&
  1226. cur->vm_userfaultfd_ctx.ctx != ctx)
  1227. goto out_unlock;
  1228. /*
  1229. * Note vmas containing huge pages
  1230. */
  1231. if (is_vm_hugetlb_page(cur))
  1232. basic_ioctls = true;
  1233. found = true;
  1234. }
  1235. BUG_ON(!found);
  1236. if (vma->vm_start < start)
  1237. prev = vma;
  1238. ret = 0;
  1239. do {
  1240. cond_resched();
  1241. BUG_ON(!vma_can_userfault(vma, vm_flags));
  1242. BUG_ON(vma->vm_userfaultfd_ctx.ctx &&
  1243. vma->vm_userfaultfd_ctx.ctx != ctx);
  1244. WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
  1245. /*
  1246. * Nothing to do: this vma is already registered into this
  1247. * userfaultfd and with the right tracking mode too.
  1248. */
  1249. if (vma->vm_userfaultfd_ctx.ctx == ctx &&
  1250. (vma->vm_flags & vm_flags) == vm_flags)
  1251. goto skip;
  1252. if (vma->vm_start > start)
  1253. start = vma->vm_start;
  1254. vma_end = min(end, vma->vm_end);
  1255. new_flags = (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags;
  1256. prev = vma_merge(mm, prev, start, vma_end, new_flags,
  1257. vma->anon_vma, vma->vm_file, vma->vm_pgoff,
  1258. vma_policy(vma),
  1259. ((struct vm_userfaultfd_ctx){ ctx }),
  1260. vma_get_anon_name(vma));
  1261. if (prev) {
  1262. vma = prev;
  1263. goto next;
  1264. }
  1265. if (vma->vm_start < start) {
  1266. ret = split_vma(mm, vma, start, 1);
  1267. if (ret)
  1268. break;
  1269. }
  1270. if (vma->vm_end > end) {
  1271. ret = split_vma(mm, vma, end, 0);
  1272. if (ret)
  1273. break;
  1274. }
  1275. next:
  1276. /*
  1277. * In the vma_merge() successful mprotect-like case 8:
  1278. * the next vma was merged into the current one and
  1279. * the current one has not been updated yet.
  1280. */
  1281. vm_write_begin(vma);
  1282. WRITE_ONCE(vma->vm_flags, new_flags);
  1283. vma->vm_userfaultfd_ctx.ctx = ctx;
  1284. vm_write_end(vma);
  1285. if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma))
  1286. hugetlb_unshare_all_pmds(vma);
  1287. skip:
  1288. prev = vma;
  1289. start = vma->vm_end;
  1290. vma = vma->vm_next;
  1291. } while (vma && vma->vm_start < end);
  1292. out_unlock:
  1293. mmap_write_unlock(mm);
  1294. mmput(mm);
  1295. if (!ret) {
  1296. __u64 ioctls_out;
  1297. ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :
  1298. UFFD_API_RANGE_IOCTLS;
  1299. /*
  1300. * Declare the WP ioctl only if the WP mode is
  1301. * specified and all checks passed with the range
  1302. */
  1303. if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP))
  1304. ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT);
  1305. /* CONTINUE ioctl is only supported for MINOR ranges. */
  1306. if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))
  1307. ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);
  1308. /*
  1309. * Now that we scanned all vmas we can already tell
  1310. * userland which ioctls methods are guaranteed to
  1311. * succeed on this range.
  1312. */
  1313. if (put_user(ioctls_out, &user_uffdio_register->ioctls))
  1314. ret = -EFAULT;
  1315. }
  1316. out:
  1317. return ret;
  1318. }
  1319. static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
  1320. unsigned long arg)
  1321. {
  1322. struct mm_struct *mm = ctx->mm;
  1323. struct vm_area_struct *vma, *prev, *cur;
  1324. int ret;
  1325. struct uffdio_range uffdio_unregister;
  1326. unsigned long new_flags;
  1327. bool found;
  1328. unsigned long start, end, vma_end;
  1329. const void __user *buf = (void __user *)arg;
  1330. ret = -EFAULT;
  1331. if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
  1332. goto out;
  1333. ret = validate_range(mm, uffdio_unregister.start,
  1334. uffdio_unregister.len);
  1335. if (ret)
  1336. goto out;
  1337. start = uffdio_unregister.start;
  1338. end = start + uffdio_unregister.len;
  1339. ret = -ENOMEM;
  1340. if (!mmget_not_zero(mm))
  1341. goto out;
  1342. mmap_write_lock(mm);
  1343. vma = find_vma_prev(mm, start, &prev);
  1344. if (!vma)
  1345. goto out_unlock;
  1346. /* check that there's at least one vma in the range */
  1347. ret = -EINVAL;
  1348. if (vma->vm_start >= end)
  1349. goto out_unlock;
  1350. /*
  1351. * If the first vma contains huge pages, make sure start address
  1352. * is aligned to huge page size.
  1353. */
  1354. if (is_vm_hugetlb_page(vma)) {
  1355. unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
  1356. if (start & (vma_hpagesize - 1))
  1357. goto out_unlock;
  1358. }
  1359. /*
  1360. * Search for not compatible vmas.
  1361. */
  1362. found = false;
  1363. ret = -EINVAL;
  1364. for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
  1365. cond_resched();
  1366. BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
  1367. !!(cur->vm_flags & __VM_UFFD_FLAGS));
  1368. /*
  1369. * Check not compatible vmas, not strictly required
  1370. * here as not compatible vmas cannot have an
  1371. * userfaultfd_ctx registered on them, but this
  1372. * provides for more strict behavior to notice
  1373. * unregistration errors.
  1374. */
  1375. if (!vma_can_userfault(cur, cur->vm_flags))
  1376. goto out_unlock;
  1377. found = true;
  1378. }
  1379. BUG_ON(!found);
  1380. if (vma->vm_start < start)
  1381. prev = vma;
  1382. ret = 0;
  1383. do {
  1384. cond_resched();
  1385. BUG_ON(!vma_can_userfault(vma, vma->vm_flags));
  1386. /*
  1387. * Nothing to do: this vma is already registered into this
  1388. * userfaultfd and with the right tracking mode too.
  1389. */
  1390. if (!vma->vm_userfaultfd_ctx.ctx)
  1391. goto skip;
  1392. WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
  1393. if (vma->vm_start > start)
  1394. start = vma->vm_start;
  1395. vma_end = min(end, vma->vm_end);
  1396. if (userfaultfd_missing(vma)) {
  1397. /*
  1398. * Wake any concurrent pending userfault while
  1399. * we unregister, so they will not hang
  1400. * permanently and it avoids userland to call
  1401. * UFFDIO_WAKE explicitly.
  1402. */
  1403. struct userfaultfd_wake_range range;
  1404. range.start = start;
  1405. range.len = vma_end - start;
  1406. wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
  1407. }
  1408. new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
  1409. prev = vma_merge(mm, prev, start, vma_end, new_flags,
  1410. vma->anon_vma, vma->vm_file, vma->vm_pgoff,
  1411. vma_policy(vma),
  1412. NULL_VM_UFFD_CTX,
  1413. vma_get_anon_name(vma));
  1414. if (prev) {
  1415. vma = prev;
  1416. goto next;
  1417. }
  1418. if (vma->vm_start < start) {
  1419. ret = split_vma(mm, vma, start, 1);
  1420. if (ret)
  1421. break;
  1422. }
  1423. if (vma->vm_end > end) {
  1424. ret = split_vma(mm, vma, end, 0);
  1425. if (ret)
  1426. break;
  1427. }
  1428. next:
  1429. /*
  1430. * In the vma_merge() successful mprotect-like case 8:
  1431. * the next vma was merged into the current one and
  1432. * the current one has not been updated yet.
  1433. */
  1434. vm_write_begin(vma);
  1435. WRITE_ONCE(vma->vm_flags, new_flags);
  1436. vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
  1437. vm_write_end(vma);
  1438. skip:
  1439. prev = vma;
  1440. start = vma->vm_end;
  1441. vma = vma->vm_next;
  1442. } while (vma && vma->vm_start < end);
  1443. out_unlock:
  1444. mmap_write_unlock(mm);
  1445. mmput(mm);
  1446. out:
  1447. return ret;
  1448. }
  1449. /*
  1450. * userfaultfd_wake may be used in combination with the
  1451. * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
  1452. */
  1453. static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
  1454. unsigned long arg)
  1455. {
  1456. int ret;
  1457. struct uffdio_range uffdio_wake;
  1458. struct userfaultfd_wake_range range;
  1459. const void __user *buf = (void __user *)arg;
  1460. ret = -EFAULT;
  1461. if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
  1462. goto out;
  1463. ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len);
  1464. if (ret)
  1465. goto out;
  1466. range.start = uffdio_wake.start;
  1467. range.len = uffdio_wake.len;
  1468. /*
  1469. * len == 0 means wake all and we don't want to wake all here,
  1470. * so check it again to be sure.
  1471. */
  1472. VM_BUG_ON(!range.len);
  1473. wake_userfault(ctx, &range);
  1474. ret = 0;
  1475. out:
  1476. return ret;
  1477. }
  1478. static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
  1479. unsigned long arg)
  1480. {
  1481. __s64 ret;
  1482. struct uffdio_copy uffdio_copy;
  1483. struct uffdio_copy __user *user_uffdio_copy;
  1484. struct userfaultfd_wake_range range;
  1485. user_uffdio_copy = (struct uffdio_copy __user *) arg;
  1486. ret = -EAGAIN;
  1487. if (READ_ONCE(ctx->mmap_changing))
  1488. goto out;
  1489. ret = -EFAULT;
  1490. if (copy_from_user(&uffdio_copy, user_uffdio_copy,
  1491. /* don't copy "copy" last field */
  1492. sizeof(uffdio_copy)-sizeof(__s64)))
  1493. goto out;
  1494. ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len);
  1495. if (ret)
  1496. goto out;
  1497. /*
  1498. * double check for wraparound just in case. copy_from_user()
  1499. * will later check uffdio_copy.src + uffdio_copy.len to fit
  1500. * in the userland range.
  1501. */
  1502. ret = -EINVAL;
  1503. if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src)
  1504. goto out;
  1505. if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP))
  1506. goto out;
  1507. if (mmget_not_zero(ctx->mm)) {
  1508. ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src,
  1509. uffdio_copy.len, &ctx->mmap_changing,
  1510. uffdio_copy.mode);
  1511. mmput(ctx->mm);
  1512. } else {
  1513. return -ESRCH;
  1514. }
  1515. if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
  1516. return -EFAULT;
  1517. if (ret < 0)
  1518. goto out;
  1519. BUG_ON(!ret);
  1520. /* len == 0 would wake all */
  1521. range.len = ret;
  1522. if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
  1523. range.start = uffdio_copy.dst;
  1524. wake_userfault(ctx, &range);
  1525. }
  1526. ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
  1527. out:
  1528. return ret;
  1529. }
  1530. static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
  1531. unsigned long arg)
  1532. {
  1533. __s64 ret;
  1534. struct uffdio_zeropage uffdio_zeropage;
  1535. struct uffdio_zeropage __user *user_uffdio_zeropage;
  1536. struct userfaultfd_wake_range range;
  1537. user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
  1538. ret = -EAGAIN;
  1539. if (READ_ONCE(ctx->mmap_changing))
  1540. goto out;
  1541. ret = -EFAULT;
  1542. if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
  1543. /* don't copy "zeropage" last field */
  1544. sizeof(uffdio_zeropage)-sizeof(__s64)))
  1545. goto out;
  1546. ret = validate_range(ctx->mm, uffdio_zeropage.range.start,
  1547. uffdio_zeropage.range.len);
  1548. if (ret)
  1549. goto out;
  1550. ret = -EINVAL;
  1551. if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
  1552. goto out;
  1553. if (mmget_not_zero(ctx->mm)) {
  1554. ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start,
  1555. uffdio_zeropage.range.len,
  1556. &ctx->mmap_changing);
  1557. mmput(ctx->mm);
  1558. } else {
  1559. return -ESRCH;
  1560. }
  1561. if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
  1562. return -EFAULT;
  1563. if (ret < 0)
  1564. goto out;
  1565. /* len == 0 would wake all */
  1566. BUG_ON(!ret);
  1567. range.len = ret;
  1568. if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
  1569. range.start = uffdio_zeropage.range.start;
  1570. wake_userfault(ctx, &range);
  1571. }
  1572. ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
  1573. out:
  1574. return ret;
  1575. }
  1576. static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx,
  1577. unsigned long arg)
  1578. {
  1579. int ret;
  1580. struct uffdio_writeprotect uffdio_wp;
  1581. struct uffdio_writeprotect __user *user_uffdio_wp;
  1582. struct userfaultfd_wake_range range;
  1583. bool mode_wp, mode_dontwake;
  1584. if (READ_ONCE(ctx->mmap_changing))
  1585. return -EAGAIN;
  1586. user_uffdio_wp = (struct uffdio_writeprotect __user *) arg;
  1587. if (copy_from_user(&uffdio_wp, user_uffdio_wp,
  1588. sizeof(struct uffdio_writeprotect)))
  1589. return -EFAULT;
  1590. ret = validate_range(ctx->mm, uffdio_wp.range.start,
  1591. uffdio_wp.range.len);
  1592. if (ret)
  1593. return ret;
  1594. if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE |
  1595. UFFDIO_WRITEPROTECT_MODE_WP))
  1596. return -EINVAL;
  1597. mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP;
  1598. mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE;
  1599. if (mode_wp && mode_dontwake)
  1600. return -EINVAL;
  1601. if (mmget_not_zero(ctx->mm)) {
  1602. ret = mwriteprotect_range(ctx->mm, uffdio_wp.range.start,
  1603. uffdio_wp.range.len, mode_wp,
  1604. &ctx->mmap_changing);
  1605. mmput(ctx->mm);
  1606. } else {
  1607. return -ESRCH;
  1608. }
  1609. if (ret)
  1610. return ret;
  1611. if (!mode_wp && !mode_dontwake) {
  1612. range.start = uffdio_wp.range.start;
  1613. range.len = uffdio_wp.range.len;
  1614. wake_userfault(ctx, &range);
  1615. }
  1616. return ret;
  1617. }
  1618. static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)
  1619. {
  1620. __s64 ret;
  1621. struct uffdio_continue uffdio_continue;
  1622. struct uffdio_continue __user *user_uffdio_continue;
  1623. struct userfaultfd_wake_range range;
  1624. user_uffdio_continue = (struct uffdio_continue __user *)arg;
  1625. ret = -EAGAIN;
  1626. if (READ_ONCE(ctx->mmap_changing))
  1627. goto out;
  1628. ret = -EFAULT;
  1629. if (copy_from_user(&uffdio_continue, user_uffdio_continue,
  1630. /* don't copy the output fields */
  1631. sizeof(uffdio_continue) - (sizeof(__s64))))
  1632. goto out;
  1633. ret = validate_range(ctx->mm, uffdio_continue.range.start,
  1634. uffdio_continue.range.len);
  1635. if (ret)
  1636. goto out;
  1637. ret = -EINVAL;
  1638. /* double check for wraparound just in case. */
  1639. if (uffdio_continue.range.start + uffdio_continue.range.len <=
  1640. uffdio_continue.range.start) {
  1641. goto out;
  1642. }
  1643. if (uffdio_continue.mode & ~UFFDIO_CONTINUE_MODE_DONTWAKE)
  1644. goto out;
  1645. if (mmget_not_zero(ctx->mm)) {
  1646. ret = mcopy_continue(ctx->mm, uffdio_continue.range.start,
  1647. uffdio_continue.range.len,
  1648. &ctx->mmap_changing);
  1649. mmput(ctx->mm);
  1650. } else {
  1651. return -ESRCH;
  1652. }
  1653. if (unlikely(put_user(ret, &user_uffdio_continue->mapped)))
  1654. return -EFAULT;
  1655. if (ret < 0)
  1656. goto out;
  1657. /* len == 0 would wake all */
  1658. BUG_ON(!ret);
  1659. range.len = ret;
  1660. if (!(uffdio_continue.mode & UFFDIO_CONTINUE_MODE_DONTWAKE)) {
  1661. range.start = uffdio_continue.range.start;
  1662. wake_userfault(ctx, &range);
  1663. }
  1664. ret = range.len == uffdio_continue.range.len ? 0 : -EAGAIN;
  1665. out:
  1666. return ret;
  1667. }
  1668. static inline unsigned int uffd_ctx_features(__u64 user_features)
  1669. {
  1670. /*
  1671. * For the current set of features the bits just coincide. Set
  1672. * UFFD_FEATURE_INITIALIZED to mark the features as enabled.
  1673. */
  1674. return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED;
  1675. }
  1676. /*
  1677. * userland asks for a certain API version and we return which bits
  1678. * and ioctl commands are implemented in this kernel for such API
  1679. * version or -EINVAL if unknown.
  1680. */
  1681. static int userfaultfd_api(struct userfaultfd_ctx *ctx,
  1682. unsigned long arg)
  1683. {
  1684. struct uffdio_api uffdio_api;
  1685. void __user *buf = (void __user *)arg;
  1686. unsigned int ctx_features;
  1687. int ret;
  1688. __u64 features;
  1689. ret = -EFAULT;
  1690. if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
  1691. goto out;
  1692. features = uffdio_api.features;
  1693. ret = -EINVAL;
  1694. if (uffdio_api.api != UFFD_API || (features & ~UFFD_API_FEATURES))
  1695. goto err_out;
  1696. ret = -EPERM;
  1697. if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE))
  1698. goto err_out;
  1699. /* report all available features and ioctls to userland */
  1700. uffdio_api.features = UFFD_API_FEATURES;
  1701. #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
  1702. uffdio_api.features &=
  1703. ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM);
  1704. #endif
  1705. uffdio_api.ioctls = UFFD_API_IOCTLS;
  1706. ret = -EFAULT;
  1707. if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
  1708. goto out;
  1709. /* only enable the requested features for this uffd context */
  1710. ctx_features = uffd_ctx_features(features);
  1711. ret = -EINVAL;
  1712. if (cmpxchg(&ctx->features, 0, ctx_features) != 0)
  1713. goto err_out;
  1714. ret = 0;
  1715. out:
  1716. return ret;
  1717. err_out:
  1718. memset(&uffdio_api, 0, sizeof(uffdio_api));
  1719. if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
  1720. ret = -EFAULT;
  1721. goto out;
  1722. }
  1723. static long userfaultfd_ioctl(struct file *file, unsigned cmd,
  1724. unsigned long arg)
  1725. {
  1726. int ret = -EINVAL;
  1727. struct userfaultfd_ctx *ctx = file->private_data;
  1728. if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx))
  1729. return -EINVAL;
  1730. switch(cmd) {
  1731. case UFFDIO_API:
  1732. ret = userfaultfd_api(ctx, arg);
  1733. break;
  1734. case UFFDIO_REGISTER:
  1735. ret = userfaultfd_register(ctx, arg);
  1736. break;
  1737. case UFFDIO_UNREGISTER:
  1738. ret = userfaultfd_unregister(ctx, arg);
  1739. break;
  1740. case UFFDIO_WAKE:
  1741. ret = userfaultfd_wake(ctx, arg);
  1742. break;
  1743. case UFFDIO_COPY:
  1744. ret = userfaultfd_copy(ctx, arg);
  1745. break;
  1746. case UFFDIO_ZEROPAGE:
  1747. ret = userfaultfd_zeropage(ctx, arg);
  1748. break;
  1749. case UFFDIO_WRITEPROTECT:
  1750. ret = userfaultfd_writeprotect(ctx, arg);
  1751. break;
  1752. case UFFDIO_CONTINUE:
  1753. ret = userfaultfd_continue(ctx, arg);
  1754. break;
  1755. }
  1756. return ret;
  1757. }
  1758. #ifdef CONFIG_PROC_FS
  1759. static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
  1760. {
  1761. struct userfaultfd_ctx *ctx = f->private_data;
  1762. wait_queue_entry_t *wq;
  1763. unsigned long pending = 0, total = 0;
  1764. spin_lock_irq(&ctx->fault_pending_wqh.lock);
  1765. list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {
  1766. pending++;
  1767. total++;
  1768. }
  1769. list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {
  1770. total++;
  1771. }
  1772. spin_unlock_irq(&ctx->fault_pending_wqh.lock);
  1773. /*
  1774. * If more protocols will be added, there will be all shown
  1775. * separated by a space. Like this:
  1776. * protocols: aa:... bb:...
  1777. */
  1778. seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
  1779. pending, total, UFFD_API, ctx->features,
  1780. UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
  1781. }
  1782. #endif
  1783. static const struct file_operations userfaultfd_fops = {
  1784. #ifdef CONFIG_PROC_FS
  1785. .show_fdinfo = userfaultfd_show_fdinfo,
  1786. #endif
  1787. .release = userfaultfd_release,
  1788. .poll = userfaultfd_poll,
  1789. .read = userfaultfd_read,
  1790. .unlocked_ioctl = userfaultfd_ioctl,
  1791. .compat_ioctl = compat_ptr_ioctl,
  1792. .llseek = noop_llseek,
  1793. };
  1794. static void init_once_userfaultfd_ctx(void *mem)
  1795. {
  1796. struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
  1797. init_waitqueue_head(&ctx->fault_pending_wqh);
  1798. init_waitqueue_head(&ctx->fault_wqh);
  1799. init_waitqueue_head(&ctx->event_wqh);
  1800. init_waitqueue_head(&ctx->fd_wqh);
  1801. seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock);
  1802. }
  1803. SYSCALL_DEFINE1(userfaultfd, int, flags)
  1804. {
  1805. struct userfaultfd_ctx *ctx;
  1806. int fd;
  1807. if (!sysctl_unprivileged_userfaultfd &&
  1808. (flags & UFFD_USER_MODE_ONLY) == 0 &&
  1809. !capable(CAP_SYS_PTRACE)) {
  1810. printk_once(KERN_WARNING "uffd: Set unprivileged_userfaultfd "
  1811. "sysctl knob to 1 if kernel faults must be handled "
  1812. "without obtaining CAP_SYS_PTRACE capability\n");
  1813. return -EPERM;
  1814. }
  1815. BUG_ON(!current->mm);
  1816. /* Check the UFFD_* constants for consistency. */
  1817. BUILD_BUG_ON(UFFD_USER_MODE_ONLY & UFFD_SHARED_FCNTL_FLAGS);
  1818. BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);
  1819. BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);
  1820. if (flags & ~(UFFD_SHARED_FCNTL_FLAGS | UFFD_USER_MODE_ONLY))
  1821. return -EINVAL;
  1822. ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
  1823. if (!ctx)
  1824. return -ENOMEM;
  1825. refcount_set(&ctx->refcount, 1);
  1826. ctx->flags = flags;
  1827. ctx->features = 0;
  1828. ctx->released = false;
  1829. ctx->mmap_changing = false;
  1830. ctx->mm = current->mm;
  1831. /* prevent the mm struct to be freed */
  1832. mmgrab(ctx->mm);
  1833. fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, ctx,
  1834. O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL);
  1835. if (fd < 0) {
  1836. mmdrop(ctx->mm);
  1837. kmem_cache_free(userfaultfd_ctx_cachep, ctx);
  1838. }
  1839. return fd;
  1840. }
  1841. static int __init userfaultfd_init(void)
  1842. {
  1843. userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
  1844. sizeof(struct userfaultfd_ctx),
  1845. 0,
  1846. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  1847. init_once_userfaultfd_ctx);
  1848. return 0;
  1849. }
  1850. __initcall(userfaultfd_init);