mmu_notifier.h 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _LINUX_MMU_NOTIFIER_H
  3. #define _LINUX_MMU_NOTIFIER_H
  4. #include <linux/list.h>
  5. #include <linux/spinlock.h>
  6. #include <linux/mm_types.h>
  7. #include <linux/mmap_lock.h>
  8. #include <linux/percpu-rwsem.h>
  9. #include <linux/slab.h>
  10. #include <linux/srcu.h>
  11. #include <linux/interval_tree.h>
  12. #include <linux/android_kabi.h>
  13. struct mmu_notifier_subscriptions;
  14. struct mmu_notifier;
  15. struct mmu_notifier_range;
  16. struct mmu_interval_notifier;
  17. struct mmu_notifier_subscriptions_hdr {
  18. bool valid;
  19. #ifdef CONFIG_SPECULATIVE_PAGE_FAULT
  20. struct percpu_rw_semaphore_atomic *mmu_notifier_lock;
  21. #endif
  22. };
  23. /**
  24. * enum mmu_notifier_event - reason for the mmu notifier callback
  25. * @MMU_NOTIFY_UNMAP: either munmap() that unmap the range or a mremap() that
  26. * move the range
  27. *
  28. * @MMU_NOTIFY_CLEAR: clear page table entry (many reasons for this like
  29. * madvise() or replacing a page by another one, ...).
  30. *
  31. * @MMU_NOTIFY_PROTECTION_VMA: update is due to protection change for the range
  32. * ie using the vma access permission (vm_page_prot) to update the whole range
  33. * is enough no need to inspect changes to the CPU page table (mprotect()
  34. * syscall)
  35. *
  36. * @MMU_NOTIFY_PROTECTION_PAGE: update is due to change in read/write flag for
  37. * pages in the range so to mirror those changes the user must inspect the CPU
  38. * page table (from the end callback).
  39. *
  40. * @MMU_NOTIFY_SOFT_DIRTY: soft dirty accounting (still same page and same
  41. * access flags). User should soft dirty the page in the end callback to make
  42. * sure that anyone relying on soft dirtyness catch pages that might be written
  43. * through non CPU mappings.
  44. *
  45. * @MMU_NOTIFY_RELEASE: used during mmu_interval_notifier invalidate to signal
  46. * that the mm refcount is zero and the range is no longer accessible.
  47. *
  48. * @MMU_NOTIFY_MIGRATE: used during migrate_vma_collect() invalidate to signal
  49. * a device driver to possibly ignore the invalidation if the
  50. * migrate_pgmap_owner field matches the driver's device private pgmap owner.
  51. */
  52. enum mmu_notifier_event {
  53. MMU_NOTIFY_UNMAP = 0,
  54. MMU_NOTIFY_CLEAR,
  55. MMU_NOTIFY_PROTECTION_VMA,
  56. MMU_NOTIFY_PROTECTION_PAGE,
  57. MMU_NOTIFY_SOFT_DIRTY,
  58. MMU_NOTIFY_RELEASE,
  59. MMU_NOTIFY_MIGRATE,
  60. };
  61. #define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)
  62. struct mmu_notifier_ops {
  63. /*
  64. * Called either by mmu_notifier_unregister or when the mm is
  65. * being destroyed by exit_mmap, always before all pages are
  66. * freed. This can run concurrently with other mmu notifier
  67. * methods (the ones invoked outside the mm context) and it
  68. * should tear down all secondary mmu mappings and freeze the
  69. * secondary mmu. If this method isn't implemented you've to
  70. * be sure that nothing could possibly write to the pages
  71. * through the secondary mmu by the time the last thread with
  72. * tsk->mm == mm exits.
  73. *
  74. * As side note: the pages freed after ->release returns could
  75. * be immediately reallocated by the gart at an alias physical
  76. * address with a different cache model, so if ->release isn't
  77. * implemented because all _software_ driven memory accesses
  78. * through the secondary mmu are terminated by the time the
  79. * last thread of this mm quits, you've also to be sure that
  80. * speculative _hardware_ operations can't allocate dirty
  81. * cachelines in the cpu that could not be snooped and made
  82. * coherent with the other read and write operations happening
  83. * through the gart alias address, so leading to memory
  84. * corruption.
  85. */
  86. void (*release)(struct mmu_notifier *subscription,
  87. struct mm_struct *mm);
  88. /*
  89. * clear_flush_young is called after the VM is
  90. * test-and-clearing the young/accessed bitflag in the
  91. * pte. This way the VM will provide proper aging to the
  92. * accesses to the page through the secondary MMUs and not
  93. * only to the ones through the Linux pte.
  94. * Start-end is necessary in case the secondary MMU is mapping the page
  95. * at a smaller granularity than the primary MMU.
  96. */
  97. int (*clear_flush_young)(struct mmu_notifier *subscription,
  98. struct mm_struct *mm,
  99. unsigned long start,
  100. unsigned long end);
  101. /*
  102. * clear_young is a lightweight version of clear_flush_young. Like the
  103. * latter, it is supposed to test-and-clear the young/accessed bitflag
  104. * in the secondary pte, but it may omit flushing the secondary tlb.
  105. */
  106. int (*clear_young)(struct mmu_notifier *subscription,
  107. struct mm_struct *mm,
  108. unsigned long start,
  109. unsigned long end);
  110. /*
  111. * test_young is called to check the young/accessed bitflag in
  112. * the secondary pte. This is used to know if the page is
  113. * frequently used without actually clearing the flag or tearing
  114. * down the secondary mapping on the page.
  115. */
  116. int (*test_young)(struct mmu_notifier *subscription,
  117. struct mm_struct *mm,
  118. unsigned long address);
  119. /*
  120. * change_pte is called in cases that pte mapping to page is changed:
  121. * for example, when ksm remaps pte to point to a new shared page.
  122. */
  123. void (*change_pte)(struct mmu_notifier *subscription,
  124. struct mm_struct *mm,
  125. unsigned long address,
  126. pte_t pte);
  127. /*
  128. * invalidate_range_start() and invalidate_range_end() must be
  129. * paired and are called only when the mmap_lock and/or the
  130. * locks protecting the reverse maps are held. If the subsystem
  131. * can't guarantee that no additional references are taken to
  132. * the pages in the range, it has to implement the
  133. * invalidate_range() notifier to remove any references taken
  134. * after invalidate_range_start().
  135. *
  136. * Invalidation of multiple concurrent ranges may be
  137. * optionally permitted by the driver. Either way the
  138. * establishment of sptes is forbidden in the range passed to
  139. * invalidate_range_begin/end for the whole duration of the
  140. * invalidate_range_begin/end critical section.
  141. *
  142. * invalidate_range_start() is called when all pages in the
  143. * range are still mapped and have at least a refcount of one.
  144. *
  145. * invalidate_range_end() is called when all pages in the
  146. * range have been unmapped and the pages have been freed by
  147. * the VM.
  148. *
  149. * The VM will remove the page table entries and potentially
  150. * the page between invalidate_range_start() and
  151. * invalidate_range_end(). If the page must not be freed
  152. * because of pending I/O or other circumstances then the
  153. * invalidate_range_start() callback (or the initial mapping
  154. * by the driver) must make sure that the refcount is kept
  155. * elevated.
  156. *
  157. * If the driver increases the refcount when the pages are
  158. * initially mapped into an address space then either
  159. * invalidate_range_start() or invalidate_range_end() may
  160. * decrease the refcount. If the refcount is decreased on
  161. * invalidate_range_start() then the VM can free pages as page
  162. * table entries are removed. If the refcount is only
  163. * droppped on invalidate_range_end() then the driver itself
  164. * will drop the last refcount but it must take care to flush
  165. * any secondary tlb before doing the final free on the
  166. * page. Pages will no longer be referenced by the linux
  167. * address space but may still be referenced by sptes until
  168. * the last refcount is dropped.
  169. *
  170. * If blockable argument is set to false then the callback cannot
  171. * sleep and has to return with -EAGAIN if sleeping would be required.
  172. * 0 should be returned otherwise. Please note that notifiers that can
  173. * fail invalidate_range_start are not allowed to implement
  174. * invalidate_range_end, as there is no mechanism for informing the
  175. * notifier that its start failed.
  176. */
  177. int (*invalidate_range_start)(struct mmu_notifier *subscription,
  178. const struct mmu_notifier_range *range);
  179. void (*invalidate_range_end)(struct mmu_notifier *subscription,
  180. const struct mmu_notifier_range *range);
  181. /*
  182. * invalidate_range() is either called between
  183. * invalidate_range_start() and invalidate_range_end() when the
  184. * VM has to free pages that where unmapped, but before the
  185. * pages are actually freed, or outside of _start()/_end() when
  186. * a (remote) TLB is necessary.
  187. *
  188. * If invalidate_range() is used to manage a non-CPU TLB with
  189. * shared page-tables, it not necessary to implement the
  190. * invalidate_range_start()/end() notifiers, as
  191. * invalidate_range() alread catches the points in time when an
  192. * external TLB range needs to be flushed. For more in depth
  193. * discussion on this see Documentation/vm/mmu_notifier.rst
  194. *
  195. * Note that this function might be called with just a sub-range
  196. * of what was passed to invalidate_range_start()/end(), if
  197. * called between those functions.
  198. */
  199. void (*invalidate_range)(struct mmu_notifier *subscription,
  200. struct mm_struct *mm,
  201. unsigned long start,
  202. unsigned long end);
  203. /*
  204. * These callbacks are used with the get/put interface to manage the
  205. * lifetime of the mmu_notifier memory. alloc_notifier() returns a new
  206. * notifier for use with the mm.
  207. *
  208. * free_notifier() is only called after the mmu_notifier has been
  209. * fully put, calls to any ops callback are prevented and no ops
  210. * callbacks are currently running. It is called from a SRCU callback
  211. * and cannot sleep.
  212. */
  213. struct mmu_notifier *(*alloc_notifier)(struct mm_struct *mm);
  214. void (*free_notifier)(struct mmu_notifier *subscription);
  215. ANDROID_KABI_RESERVE(1);
  216. ANDROID_KABI_RESERVE(2);
  217. ANDROID_KABI_RESERVE(3);
  218. ANDROID_KABI_RESERVE(4);
  219. };
  220. /*
  221. * The notifier chains are protected by mmap_lock and/or the reverse map
  222. * semaphores. Notifier chains are only changed when all reverse maps and
  223. * the mmap_lock locks are taken.
  224. *
  225. * Therefore notifier chains can only be traversed when either
  226. *
  227. * 1. mmap_lock is held.
  228. * 2. One of the reverse map locks is held (i_mmap_rwsem or anon_vma->rwsem).
  229. * 3. No other concurrent thread can access the list (release)
  230. */
  231. struct mmu_notifier {
  232. struct hlist_node hlist;
  233. const struct mmu_notifier_ops *ops;
  234. struct mm_struct *mm;
  235. struct rcu_head rcu;
  236. unsigned int users;
  237. ANDROID_KABI_RESERVE(1);
  238. ANDROID_KABI_RESERVE(2);
  239. };
  240. /**
  241. * struct mmu_interval_notifier_ops
  242. * @invalidate: Upon return the caller must stop using any SPTEs within this
  243. * range. This function can sleep. Return false only if sleeping
  244. * was required but mmu_notifier_range_blockable(range) is false.
  245. */
  246. struct mmu_interval_notifier_ops {
  247. bool (*invalidate)(struct mmu_interval_notifier *interval_sub,
  248. const struct mmu_notifier_range *range,
  249. unsigned long cur_seq);
  250. };
  251. struct mmu_interval_notifier {
  252. struct interval_tree_node interval_tree;
  253. const struct mmu_interval_notifier_ops *ops;
  254. struct mm_struct *mm;
  255. struct hlist_node deferred_item;
  256. unsigned long invalidate_seq;
  257. };
  258. #ifdef CONFIG_MMU_NOTIFIER
  259. #ifdef CONFIG_LOCKDEP
  260. extern struct lockdep_map __mmu_notifier_invalidate_range_start_map;
  261. #endif
  262. struct mmu_notifier_range {
  263. struct vm_area_struct *vma;
  264. struct mm_struct *mm;
  265. unsigned long start;
  266. unsigned long end;
  267. unsigned flags;
  268. enum mmu_notifier_event event;
  269. void *migrate_pgmap_owner;
  270. };
  271. static inline
  272. struct mmu_notifier_subscriptions_hdr *get_notifier_subscriptions_hdr(
  273. struct mm_struct *mm)
  274. {
  275. /*
  276. * container_of() can't be used here because mmu_notifier_subscriptions
  277. * struct should be kept invisible to mm_struct, otherwise it
  278. * introduces KMI CRC breakage. Therefore the callers don't know what
  279. * members struct mmu_notifier_subscriptions contains and can't call
  280. * container_of(), which requires a member name.
  281. *
  282. * WARNING: For this typecasting to work, mmu_notifier_subscriptions_hdr
  283. * should be the first member of struct mmu_notifier_subscriptions.
  284. */
  285. return (struct mmu_notifier_subscriptions_hdr *)mm->notifier_subscriptions;
  286. }
  287. static inline int mm_has_notifiers(struct mm_struct *mm)
  288. {
  289. #ifdef CONFIG_SPECULATIVE_PAGE_FAULT
  290. return unlikely(get_notifier_subscriptions_hdr(mm)->valid);
  291. #else
  292. return unlikely(mm->notifier_subscriptions);
  293. #endif
  294. }
  295. struct mmu_notifier *mmu_notifier_get_locked(const struct mmu_notifier_ops *ops,
  296. struct mm_struct *mm);
  297. static inline struct mmu_notifier *
  298. mmu_notifier_get(const struct mmu_notifier_ops *ops, struct mm_struct *mm)
  299. {
  300. struct mmu_notifier *ret;
  301. mmap_write_lock(mm);
  302. ret = mmu_notifier_get_locked(ops, mm);
  303. mmap_write_unlock(mm);
  304. return ret;
  305. }
  306. void mmu_notifier_put(struct mmu_notifier *subscription);
  307. void mmu_notifier_synchronize(void);
  308. extern int mmu_notifier_register(struct mmu_notifier *subscription,
  309. struct mm_struct *mm);
  310. extern int __mmu_notifier_register(struct mmu_notifier *subscription,
  311. struct mm_struct *mm);
  312. extern void mmu_notifier_unregister(struct mmu_notifier *subscription,
  313. struct mm_struct *mm);
  314. unsigned long
  315. mmu_interval_read_begin(struct mmu_interval_notifier *interval_sub);
  316. int mmu_interval_notifier_insert(struct mmu_interval_notifier *interval_sub,
  317. struct mm_struct *mm, unsigned long start,
  318. unsigned long length,
  319. const struct mmu_interval_notifier_ops *ops);
  320. int mmu_interval_notifier_insert_locked(
  321. struct mmu_interval_notifier *interval_sub, struct mm_struct *mm,
  322. unsigned long start, unsigned long length,
  323. const struct mmu_interval_notifier_ops *ops);
  324. void mmu_interval_notifier_remove(struct mmu_interval_notifier *interval_sub);
  325. /**
  326. * mmu_interval_set_seq - Save the invalidation sequence
  327. * @interval_sub - The subscription passed to invalidate
  328. * @cur_seq - The cur_seq passed to the invalidate() callback
  329. *
  330. * This must be called unconditionally from the invalidate callback of a
  331. * struct mmu_interval_notifier_ops under the same lock that is used to call
  332. * mmu_interval_read_retry(). It updates the sequence number for later use by
  333. * mmu_interval_read_retry(). The provided cur_seq will always be odd.
  334. *
  335. * If the caller does not call mmu_interval_read_begin() or
  336. * mmu_interval_read_retry() then this call is not required.
  337. */
  338. static inline void
  339. mmu_interval_set_seq(struct mmu_interval_notifier *interval_sub,
  340. unsigned long cur_seq)
  341. {
  342. WRITE_ONCE(interval_sub->invalidate_seq, cur_seq);
  343. }
  344. /**
  345. * mmu_interval_read_retry - End a read side critical section against a VA range
  346. * interval_sub: The subscription
  347. * seq: The return of the paired mmu_interval_read_begin()
  348. *
  349. * This MUST be called under a user provided lock that is also held
  350. * unconditionally by op->invalidate() when it calls mmu_interval_set_seq().
  351. *
  352. * Each call should be paired with a single mmu_interval_read_begin() and
  353. * should be used to conclude the read side.
  354. *
  355. * Returns true if an invalidation collided with this critical section, and
  356. * the caller should retry.
  357. */
  358. static inline bool
  359. mmu_interval_read_retry(struct mmu_interval_notifier *interval_sub,
  360. unsigned long seq)
  361. {
  362. return interval_sub->invalidate_seq != seq;
  363. }
  364. /**
  365. * mmu_interval_check_retry - Test if a collision has occurred
  366. * interval_sub: The subscription
  367. * seq: The return of the matching mmu_interval_read_begin()
  368. *
  369. * This can be used in the critical section between mmu_interval_read_begin()
  370. * and mmu_interval_read_retry(). A return of true indicates an invalidation
  371. * has collided with this critical region and a future
  372. * mmu_interval_read_retry() will return true.
  373. *
  374. * False is not reliable and only suggests a collision may not have
  375. * occured. It can be called many times and does not have to hold the user
  376. * provided lock.
  377. *
  378. * This call can be used as part of loops and other expensive operations to
  379. * expedite a retry.
  380. */
  381. static inline bool
  382. mmu_interval_check_retry(struct mmu_interval_notifier *interval_sub,
  383. unsigned long seq)
  384. {
  385. /* Pairs with the WRITE_ONCE in mmu_interval_set_seq() */
  386. return READ_ONCE(interval_sub->invalidate_seq) != seq;
  387. }
  388. extern void __mmu_notifier_subscriptions_destroy(struct mm_struct *mm);
  389. extern void __mmu_notifier_release(struct mm_struct *mm);
  390. extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
  391. unsigned long start,
  392. unsigned long end);
  393. extern int __mmu_notifier_clear_young(struct mm_struct *mm,
  394. unsigned long start,
  395. unsigned long end);
  396. extern int __mmu_notifier_test_young(struct mm_struct *mm,
  397. unsigned long address);
  398. extern void __mmu_notifier_change_pte(struct mm_struct *mm,
  399. unsigned long address, pte_t pte);
  400. extern int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *r);
  401. extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r,
  402. bool only_end);
  403. extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
  404. unsigned long start, unsigned long end);
  405. extern bool
  406. mmu_notifier_range_update_to_read_only(const struct mmu_notifier_range *range);
  407. static inline bool
  408. mmu_notifier_range_blockable(const struct mmu_notifier_range *range)
  409. {
  410. return (range->flags & MMU_NOTIFIER_RANGE_BLOCKABLE);
  411. }
  412. static inline void mmu_notifier_release(struct mm_struct *mm)
  413. {
  414. if (mm_has_notifiers(mm))
  415. __mmu_notifier_release(mm);
  416. }
  417. static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
  418. unsigned long start,
  419. unsigned long end)
  420. {
  421. if (mm_has_notifiers(mm))
  422. return __mmu_notifier_clear_flush_young(mm, start, end);
  423. return 0;
  424. }
  425. static inline int mmu_notifier_clear_young(struct mm_struct *mm,
  426. unsigned long start,
  427. unsigned long end)
  428. {
  429. if (mm_has_notifiers(mm))
  430. return __mmu_notifier_clear_young(mm, start, end);
  431. return 0;
  432. }
  433. static inline int mmu_notifier_test_young(struct mm_struct *mm,
  434. unsigned long address)
  435. {
  436. if (mm_has_notifiers(mm))
  437. return __mmu_notifier_test_young(mm, address);
  438. return 0;
  439. }
  440. static inline void mmu_notifier_change_pte(struct mm_struct *mm,
  441. unsigned long address, pte_t pte)
  442. {
  443. if (mm_has_notifiers(mm))
  444. __mmu_notifier_change_pte(mm, address, pte);
  445. }
  446. static inline void
  447. mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
  448. {
  449. might_sleep();
  450. lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
  451. if (mm_has_notifiers(range->mm)) {
  452. range->flags |= MMU_NOTIFIER_RANGE_BLOCKABLE;
  453. __mmu_notifier_invalidate_range_start(range);
  454. }
  455. lock_map_release(&__mmu_notifier_invalidate_range_start_map);
  456. }
  457. static inline int
  458. mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
  459. {
  460. int ret = 0;
  461. lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
  462. if (mm_has_notifiers(range->mm)) {
  463. range->flags &= ~MMU_NOTIFIER_RANGE_BLOCKABLE;
  464. ret = __mmu_notifier_invalidate_range_start(range);
  465. }
  466. lock_map_release(&__mmu_notifier_invalidate_range_start_map);
  467. return ret;
  468. }
  469. static inline void
  470. mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
  471. {
  472. if (mmu_notifier_range_blockable(range))
  473. might_sleep();
  474. if (mm_has_notifiers(range->mm))
  475. __mmu_notifier_invalidate_range_end(range, false);
  476. }
  477. static inline void
  478. mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
  479. {
  480. if (mm_has_notifiers(range->mm))
  481. __mmu_notifier_invalidate_range_end(range, true);
  482. }
  483. static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
  484. unsigned long start, unsigned long end)
  485. {
  486. if (mm_has_notifiers(mm))
  487. __mmu_notifier_invalidate_range(mm, start, end);
  488. }
  489. #ifdef CONFIG_SPECULATIVE_PAGE_FAULT
  490. extern bool mmu_notifier_subscriptions_init(struct mm_struct *mm);
  491. extern void mmu_notifier_subscriptions_destroy(struct mm_struct *mm);
  492. static inline bool mmu_notifier_trylock(struct mm_struct *mm)
  493. {
  494. return percpu_down_read_trylock(
  495. &get_notifier_subscriptions_hdr(mm)->mmu_notifier_lock->rw_sem);
  496. }
  497. static inline void mmu_notifier_unlock(struct mm_struct *mm)
  498. {
  499. percpu_up_read(
  500. &get_notifier_subscriptions_hdr(mm)->mmu_notifier_lock->rw_sem);
  501. }
  502. #else /* CONFIG_SPECULATIVE_PAGE_FAULT */
  503. static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm)
  504. {
  505. mm->notifier_subscriptions = NULL;
  506. return true;
  507. }
  508. static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
  509. {
  510. if (mm_has_notifiers(mm))
  511. __mmu_notifier_subscriptions_destroy(mm);
  512. }
  513. static inline bool mmu_notifier_trylock(struct mm_struct *mm)
  514. {
  515. return true;
  516. }
  517. static inline void mmu_notifier_unlock(struct mm_struct *mm)
  518. {
  519. }
  520. #endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
  521. static inline void mmu_notifier_range_init(struct mmu_notifier_range *range,
  522. enum mmu_notifier_event event,
  523. unsigned flags,
  524. struct vm_area_struct *vma,
  525. struct mm_struct *mm,
  526. unsigned long start,
  527. unsigned long end)
  528. {
  529. range->vma = vma;
  530. range->event = event;
  531. range->mm = mm;
  532. range->start = start;
  533. range->end = end;
  534. range->flags = flags;
  535. }
  536. static inline void mmu_notifier_range_init_migrate(
  537. struct mmu_notifier_range *range, unsigned int flags,
  538. struct vm_area_struct *vma, struct mm_struct *mm,
  539. unsigned long start, unsigned long end, void *pgmap)
  540. {
  541. mmu_notifier_range_init(range, MMU_NOTIFY_MIGRATE, flags, vma, mm,
  542. start, end);
  543. range->migrate_pgmap_owner = pgmap;
  544. }
  545. #define ptep_clear_flush_young_notify(__vma, __address, __ptep) \
  546. ({ \
  547. int __young; \
  548. struct vm_area_struct *___vma = __vma; \
  549. unsigned long ___address = __address; \
  550. __young = ptep_clear_flush_young(___vma, ___address, __ptep); \
  551. __young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \
  552. ___address, \
  553. ___address + \
  554. PAGE_SIZE); \
  555. __young; \
  556. })
  557. #define pmdp_clear_flush_young_notify(__vma, __address, __pmdp) \
  558. ({ \
  559. int __young; \
  560. struct vm_area_struct *___vma = __vma; \
  561. unsigned long ___address = __address; \
  562. __young = pmdp_clear_flush_young(___vma, ___address, __pmdp); \
  563. __young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \
  564. ___address, \
  565. ___address + \
  566. PMD_SIZE); \
  567. __young; \
  568. })
  569. #define ptep_clear_young_notify(__vma, __address, __ptep) \
  570. ({ \
  571. int __young; \
  572. struct vm_area_struct *___vma = __vma; \
  573. unsigned long ___address = __address; \
  574. __young = ptep_test_and_clear_young(___vma, ___address, __ptep);\
  575. __young |= mmu_notifier_clear_young(___vma->vm_mm, ___address, \
  576. ___address + PAGE_SIZE); \
  577. __young; \
  578. })
  579. #define pmdp_clear_young_notify(__vma, __address, __pmdp) \
  580. ({ \
  581. int __young; \
  582. struct vm_area_struct *___vma = __vma; \
  583. unsigned long ___address = __address; \
  584. __young = pmdp_test_and_clear_young(___vma, ___address, __pmdp);\
  585. __young |= mmu_notifier_clear_young(___vma->vm_mm, ___address, \
  586. ___address + PMD_SIZE); \
  587. __young; \
  588. })
  589. #define ptep_clear_flush_notify(__vma, __address, __ptep) \
  590. ({ \
  591. unsigned long ___addr = __address & PAGE_MASK; \
  592. struct mm_struct *___mm = (__vma)->vm_mm; \
  593. pte_t ___pte; \
  594. \
  595. ___pte = ptep_clear_flush(__vma, __address, __ptep); \
  596. mmu_notifier_invalidate_range(___mm, ___addr, \
  597. ___addr + PAGE_SIZE); \
  598. \
  599. ___pte; \
  600. })
  601. #define pmdp_huge_clear_flush_notify(__vma, __haddr, __pmd) \
  602. ({ \
  603. unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \
  604. struct mm_struct *___mm = (__vma)->vm_mm; \
  605. pmd_t ___pmd; \
  606. \
  607. ___pmd = pmdp_huge_clear_flush(__vma, __haddr, __pmd); \
  608. mmu_notifier_invalidate_range(___mm, ___haddr, \
  609. ___haddr + HPAGE_PMD_SIZE); \
  610. \
  611. ___pmd; \
  612. })
  613. #define pudp_huge_clear_flush_notify(__vma, __haddr, __pud) \
  614. ({ \
  615. unsigned long ___haddr = __haddr & HPAGE_PUD_MASK; \
  616. struct mm_struct *___mm = (__vma)->vm_mm; \
  617. pud_t ___pud; \
  618. \
  619. ___pud = pudp_huge_clear_flush(__vma, __haddr, __pud); \
  620. mmu_notifier_invalidate_range(___mm, ___haddr, \
  621. ___haddr + HPAGE_PUD_SIZE); \
  622. \
  623. ___pud; \
  624. })
  625. /*
  626. * set_pte_at_notify() sets the pte _after_ running the notifier.
  627. * This is safe to start by updating the secondary MMUs, because the primary MMU
  628. * pte invalidate must have already happened with a ptep_clear_flush() before
  629. * set_pte_at_notify() has been invoked. Updating the secondary MMUs first is
  630. * required when we change both the protection of the mapping from read-only to
  631. * read-write and the pfn (like during copy on write page faults). Otherwise the
  632. * old page would remain mapped readonly in the secondary MMUs after the new
  633. * page is already writable by some CPU through the primary MMU.
  634. */
  635. #define set_pte_at_notify(__mm, __address, __ptep, __pte) \
  636. ({ \
  637. struct mm_struct *___mm = __mm; \
  638. unsigned long ___address = __address; \
  639. pte_t ___pte = __pte; \
  640. \
  641. mmu_notifier_change_pte(___mm, ___address, ___pte); \
  642. set_pte_at(___mm, ___address, __ptep, ___pte); \
  643. })
  644. #else /* CONFIG_MMU_NOTIFIER */
  645. struct mmu_notifier_range {
  646. unsigned long start;
  647. unsigned long end;
  648. };
  649. static inline void _mmu_notifier_range_init(struct mmu_notifier_range *range,
  650. unsigned long start,
  651. unsigned long end)
  652. {
  653. range->start = start;
  654. range->end = end;
  655. }
  656. #define mmu_notifier_range_init(range,event,flags,vma,mm,start,end) \
  657. _mmu_notifier_range_init(range, start, end)
  658. #define mmu_notifier_range_init_migrate(range, flags, vma, mm, start, end, \
  659. pgmap) \
  660. _mmu_notifier_range_init(range, start, end)
  661. static inline bool
  662. mmu_notifier_range_blockable(const struct mmu_notifier_range *range)
  663. {
  664. return true;
  665. }
  666. static inline int mm_has_notifiers(struct mm_struct *mm)
  667. {
  668. return 0;
  669. }
  670. static inline void mmu_notifier_release(struct mm_struct *mm)
  671. {
  672. }
  673. static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
  674. unsigned long start,
  675. unsigned long end)
  676. {
  677. return 0;
  678. }
  679. static inline int mmu_notifier_test_young(struct mm_struct *mm,
  680. unsigned long address)
  681. {
  682. return 0;
  683. }
  684. static inline void mmu_notifier_change_pte(struct mm_struct *mm,
  685. unsigned long address, pte_t pte)
  686. {
  687. }
  688. static inline void
  689. mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
  690. {
  691. }
  692. static inline int
  693. mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
  694. {
  695. return 0;
  696. }
  697. static inline
  698. void mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
  699. {
  700. }
  701. static inline void
  702. mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
  703. {
  704. }
  705. static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
  706. unsigned long start, unsigned long end)
  707. {
  708. }
  709. static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm)
  710. {
  711. return true;
  712. }
  713. static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
  714. {
  715. }
  716. static inline bool mmu_notifier_trylock(struct mm_struct *mm)
  717. {
  718. return true;
  719. }
  720. static inline void mmu_notifier_unlock(struct mm_struct *mm)
  721. {
  722. }
  723. #define mmu_notifier_range_update_to_read_only(r) false
  724. #define ptep_clear_flush_young_notify ptep_clear_flush_young
  725. #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
  726. #define ptep_clear_young_notify ptep_test_and_clear_young
  727. #define pmdp_clear_young_notify pmdp_test_and_clear_young
  728. #define ptep_clear_flush_notify ptep_clear_flush
  729. #define pmdp_huge_clear_flush_notify pmdp_huge_clear_flush
  730. #define pudp_huge_clear_flush_notify pudp_huge_clear_flush
  731. #define set_pte_at_notify set_pte_at
  732. static inline void mmu_notifier_synchronize(void)
  733. {
  734. }
  735. #endif /* CONFIG_MMU_NOTIFIER */
  736. #endif /* _LINUX_MMU_NOTIFIER_H */