dax.c 47 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * fs/dax.c - Direct Access filesystem code
  4. * Copyright (c) 2013-2014 Intel Corporation
  5. * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
  6. * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
  7. */
  8. #include <linux/atomic.h>
  9. #include <linux/blkdev.h>
  10. #include <linux/buffer_head.h>
  11. #include <linux/dax.h>
  12. #include <linux/fs.h>
  13. #include <linux/genhd.h>
  14. #include <linux/highmem.h>
  15. #include <linux/memcontrol.h>
  16. #include <linux/mm.h>
  17. #include <linux/mutex.h>
  18. #include <linux/pagevec.h>
  19. #include <linux/sched.h>
  20. #include <linux/sched/signal.h>
  21. #include <linux/uio.h>
  22. #include <linux/vmstat.h>
  23. #include <linux/pfn_t.h>
  24. #include <linux/sizes.h>
  25. #include <linux/mmu_notifier.h>
  26. #include <linux/iomap.h>
  27. #include <asm/pgalloc.h>
  28. #define CREATE_TRACE_POINTS
  29. #include <trace/events/fs_dax.h>
  30. static inline unsigned int pe_order(enum page_entry_size pe_size)
  31. {
  32. if (pe_size == PE_SIZE_PTE)
  33. return PAGE_SHIFT - PAGE_SHIFT;
  34. if (pe_size == PE_SIZE_PMD)
  35. return PMD_SHIFT - PAGE_SHIFT;
  36. if (pe_size == PE_SIZE_PUD)
  37. return PUD_SHIFT - PAGE_SHIFT;
  38. return ~0;
  39. }
  40. /* We choose 4096 entries - same as per-zone page wait tables */
  41. #define DAX_WAIT_TABLE_BITS 12
  42. #define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
  43. /* The 'colour' (ie low bits) within a PMD of a page offset. */
  44. #define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)
  45. #define PG_PMD_NR (PMD_SIZE >> PAGE_SHIFT)
  46. /* The order of a PMD entry */
  47. #define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT)
  48. static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
  49. static int __init init_dax_wait_table(void)
  50. {
  51. int i;
  52. for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
  53. init_waitqueue_head(wait_table + i);
  54. return 0;
  55. }
  56. fs_initcall(init_dax_wait_table);
  57. /*
  58. * DAX pagecache entries use XArray value entries so they can't be mistaken
  59. * for pages. We use one bit for locking, one bit for the entry size (PMD)
  60. * and two more to tell us if the entry is a zero page or an empty entry that
  61. * is just used for locking. In total four special bits.
  62. *
  63. * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
  64. * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
  65. * block allocation.
  66. */
  67. #define DAX_SHIFT (4)
  68. #define DAX_LOCKED (1UL << 0)
  69. #define DAX_PMD (1UL << 1)
  70. #define DAX_ZERO_PAGE (1UL << 2)
  71. #define DAX_EMPTY (1UL << 3)
  72. static unsigned long dax_to_pfn(void *entry)
  73. {
  74. return xa_to_value(entry) >> DAX_SHIFT;
  75. }
  76. static void *dax_make_entry(pfn_t pfn, unsigned long flags)
  77. {
  78. return xa_mk_value(flags | (pfn_t_to_pfn(pfn) << DAX_SHIFT));
  79. }
  80. static bool dax_is_locked(void *entry)
  81. {
  82. return xa_to_value(entry) & DAX_LOCKED;
  83. }
  84. static unsigned int dax_entry_order(void *entry)
  85. {
  86. if (xa_to_value(entry) & DAX_PMD)
  87. return PMD_ORDER;
  88. return 0;
  89. }
  90. static unsigned long dax_is_pmd_entry(void *entry)
  91. {
  92. return xa_to_value(entry) & DAX_PMD;
  93. }
  94. static bool dax_is_pte_entry(void *entry)
  95. {
  96. return !(xa_to_value(entry) & DAX_PMD);
  97. }
  98. static int dax_is_zero_entry(void *entry)
  99. {
  100. return xa_to_value(entry) & DAX_ZERO_PAGE;
  101. }
  102. static int dax_is_empty_entry(void *entry)
  103. {
  104. return xa_to_value(entry) & DAX_EMPTY;
  105. }
  106. /*
  107. * true if the entry that was found is of a smaller order than the entry
  108. * we were looking for
  109. */
  110. static bool dax_is_conflict(void *entry)
  111. {
  112. return entry == XA_RETRY_ENTRY;
  113. }
  114. /*
  115. * DAX page cache entry locking
  116. */
  117. struct exceptional_entry_key {
  118. struct xarray *xa;
  119. pgoff_t entry_start;
  120. };
  121. struct wait_exceptional_entry_queue {
  122. wait_queue_entry_t wait;
  123. struct exceptional_entry_key key;
  124. };
  125. /**
  126. * enum dax_wake_mode: waitqueue wakeup behaviour
  127. * @WAKE_ALL: wake all waiters in the waitqueue
  128. * @WAKE_NEXT: wake only the first waiter in the waitqueue
  129. */
  130. enum dax_wake_mode {
  131. WAKE_ALL,
  132. WAKE_NEXT,
  133. };
  134. static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas,
  135. void *entry, struct exceptional_entry_key *key)
  136. {
  137. unsigned long hash;
  138. unsigned long index = xas->xa_index;
  139. /*
  140. * If 'entry' is a PMD, align the 'index' that we use for the wait
  141. * queue to the start of that PMD. This ensures that all offsets in
  142. * the range covered by the PMD map to the same bit lock.
  143. */
  144. if (dax_is_pmd_entry(entry))
  145. index &= ~PG_PMD_COLOUR;
  146. key->xa = xas->xa;
  147. key->entry_start = index;
  148. hash = hash_long((unsigned long)xas->xa ^ index, DAX_WAIT_TABLE_BITS);
  149. return wait_table + hash;
  150. }
  151. static int wake_exceptional_entry_func(wait_queue_entry_t *wait,
  152. unsigned int mode, int sync, void *keyp)
  153. {
  154. struct exceptional_entry_key *key = keyp;
  155. struct wait_exceptional_entry_queue *ewait =
  156. container_of(wait, struct wait_exceptional_entry_queue, wait);
  157. if (key->xa != ewait->key.xa ||
  158. key->entry_start != ewait->key.entry_start)
  159. return 0;
  160. return autoremove_wake_function(wait, mode, sync, NULL);
  161. }
  162. /*
  163. * @entry may no longer be the entry at the index in the mapping.
  164. * The important information it's conveying is whether the entry at
  165. * this index used to be a PMD entry.
  166. */
  167. static void dax_wake_entry(struct xa_state *xas, void *entry,
  168. enum dax_wake_mode mode)
  169. {
  170. struct exceptional_entry_key key;
  171. wait_queue_head_t *wq;
  172. wq = dax_entry_waitqueue(xas, entry, &key);
  173. /*
  174. * Checking for locked entry and prepare_to_wait_exclusive() happens
  175. * under the i_pages lock, ditto for entry handling in our callers.
  176. * So at this point all tasks that could have seen our entry locked
  177. * must be in the waitqueue and the following check will see them.
  178. */
  179. if (waitqueue_active(wq))
  180. __wake_up(wq, TASK_NORMAL, mode == WAKE_ALL ? 0 : 1, &key);
  181. }
  182. /*
  183. * Look up entry in page cache, wait for it to become unlocked if it
  184. * is a DAX entry and return it. The caller must subsequently call
  185. * put_unlocked_entry() if it did not lock the entry or dax_unlock_entry()
  186. * if it did. The entry returned may have a larger order than @order.
  187. * If @order is larger than the order of the entry found in i_pages, this
  188. * function returns a dax_is_conflict entry.
  189. *
  190. * Must be called with the i_pages lock held.
  191. */
  192. static void *get_unlocked_entry(struct xa_state *xas, unsigned int order)
  193. {
  194. void *entry;
  195. struct wait_exceptional_entry_queue ewait;
  196. wait_queue_head_t *wq;
  197. init_wait(&ewait.wait);
  198. ewait.wait.func = wake_exceptional_entry_func;
  199. for (;;) {
  200. entry = xas_find_conflict(xas);
  201. if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
  202. return entry;
  203. if (dax_entry_order(entry) < order)
  204. return XA_RETRY_ENTRY;
  205. if (!dax_is_locked(entry))
  206. return entry;
  207. wq = dax_entry_waitqueue(xas, entry, &ewait.key);
  208. prepare_to_wait_exclusive(wq, &ewait.wait,
  209. TASK_UNINTERRUPTIBLE);
  210. xas_unlock_irq(xas);
  211. xas_reset(xas);
  212. schedule();
  213. finish_wait(wq, &ewait.wait);
  214. xas_lock_irq(xas);
  215. }
  216. }
  217. /*
  218. * The only thing keeping the address space around is the i_pages lock
  219. * (it's cycled in clear_inode() after removing the entries from i_pages)
  220. * After we call xas_unlock_irq(), we cannot touch xas->xa.
  221. */
  222. static void wait_entry_unlocked(struct xa_state *xas, void *entry)
  223. {
  224. struct wait_exceptional_entry_queue ewait;
  225. wait_queue_head_t *wq;
  226. init_wait(&ewait.wait);
  227. ewait.wait.func = wake_exceptional_entry_func;
  228. wq = dax_entry_waitqueue(xas, entry, &ewait.key);
  229. /*
  230. * Unlike get_unlocked_entry() there is no guarantee that this
  231. * path ever successfully retrieves an unlocked entry before an
  232. * inode dies. Perform a non-exclusive wait in case this path
  233. * never successfully performs its own wake up.
  234. */
  235. prepare_to_wait(wq, &ewait.wait, TASK_UNINTERRUPTIBLE);
  236. xas_unlock_irq(xas);
  237. schedule();
  238. finish_wait(wq, &ewait.wait);
  239. }
  240. static void put_unlocked_entry(struct xa_state *xas, void *entry,
  241. enum dax_wake_mode mode)
  242. {
  243. if (entry && !dax_is_conflict(entry))
  244. dax_wake_entry(xas, entry, mode);
  245. }
  246. /*
  247. * We used the xa_state to get the entry, but then we locked the entry and
  248. * dropped the xa_lock, so we know the xa_state is stale and must be reset
  249. * before use.
  250. */
  251. static void dax_unlock_entry(struct xa_state *xas, void *entry)
  252. {
  253. void *old;
  254. BUG_ON(dax_is_locked(entry));
  255. xas_reset(xas);
  256. xas_lock_irq(xas);
  257. old = xas_store(xas, entry);
  258. xas_unlock_irq(xas);
  259. BUG_ON(!dax_is_locked(old));
  260. dax_wake_entry(xas, entry, WAKE_NEXT);
  261. }
  262. /*
  263. * Return: The entry stored at this location before it was locked.
  264. */
  265. static void *dax_lock_entry(struct xa_state *xas, void *entry)
  266. {
  267. unsigned long v = xa_to_value(entry);
  268. return xas_store(xas, xa_mk_value(v | DAX_LOCKED));
  269. }
  270. static unsigned long dax_entry_size(void *entry)
  271. {
  272. if (dax_is_zero_entry(entry))
  273. return 0;
  274. else if (dax_is_empty_entry(entry))
  275. return 0;
  276. else if (dax_is_pmd_entry(entry))
  277. return PMD_SIZE;
  278. else
  279. return PAGE_SIZE;
  280. }
  281. static unsigned long dax_end_pfn(void *entry)
  282. {
  283. return dax_to_pfn(entry) + dax_entry_size(entry) / PAGE_SIZE;
  284. }
  285. /*
  286. * Iterate through all mapped pfns represented by an entry, i.e. skip
  287. * 'empty' and 'zero' entries.
  288. */
  289. #define for_each_mapped_pfn(entry, pfn) \
  290. for (pfn = dax_to_pfn(entry); \
  291. pfn < dax_end_pfn(entry); pfn++)
  292. /*
  293. * TODO: for reflink+dax we need a way to associate a single page with
  294. * multiple address_space instances at different linear_page_index()
  295. * offsets.
  296. */
  297. static void dax_associate_entry(void *entry, struct address_space *mapping,
  298. struct vm_area_struct *vma, unsigned long address)
  299. {
  300. unsigned long size = dax_entry_size(entry), pfn, index;
  301. int i = 0;
  302. if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
  303. return;
  304. index = linear_page_index(vma, address & ~(size - 1));
  305. for_each_mapped_pfn(entry, pfn) {
  306. struct page *page = pfn_to_page(pfn);
  307. WARN_ON_ONCE(page->mapping);
  308. page->mapping = mapping;
  309. page->index = index + i++;
  310. }
  311. }
  312. static void dax_disassociate_entry(void *entry, struct address_space *mapping,
  313. bool trunc)
  314. {
  315. unsigned long pfn;
  316. if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
  317. return;
  318. for_each_mapped_pfn(entry, pfn) {
  319. struct page *page = pfn_to_page(pfn);
  320. WARN_ON_ONCE(trunc && page_ref_count(page) > 1);
  321. WARN_ON_ONCE(page->mapping && page->mapping != mapping);
  322. page->mapping = NULL;
  323. page->index = 0;
  324. }
  325. }
  326. static struct page *dax_busy_page(void *entry)
  327. {
  328. unsigned long pfn;
  329. for_each_mapped_pfn(entry, pfn) {
  330. struct page *page = pfn_to_page(pfn);
  331. if (page_ref_count(page) > 1)
  332. return page;
  333. }
  334. return NULL;
  335. }
  336. /*
  337. * dax_lock_mapping_entry - Lock the DAX entry corresponding to a page
  338. * @page: The page whose entry we want to lock
  339. *
  340. * Context: Process context.
  341. * Return: A cookie to pass to dax_unlock_page() or 0 if the entry could
  342. * not be locked.
  343. */
  344. dax_entry_t dax_lock_page(struct page *page)
  345. {
  346. XA_STATE(xas, NULL, 0);
  347. void *entry;
  348. /* Ensure page->mapping isn't freed while we look at it */
  349. rcu_read_lock();
  350. for (;;) {
  351. struct address_space *mapping = READ_ONCE(page->mapping);
  352. entry = NULL;
  353. if (!mapping || !dax_mapping(mapping))
  354. break;
  355. /*
  356. * In the device-dax case there's no need to lock, a
  357. * struct dev_pagemap pin is sufficient to keep the
  358. * inode alive, and we assume we have dev_pagemap pin
  359. * otherwise we would not have a valid pfn_to_page()
  360. * translation.
  361. */
  362. entry = (void *)~0UL;
  363. if (S_ISCHR(mapping->host->i_mode))
  364. break;
  365. xas.xa = &mapping->i_pages;
  366. xas_lock_irq(&xas);
  367. if (mapping != page->mapping) {
  368. xas_unlock_irq(&xas);
  369. continue;
  370. }
  371. xas_set(&xas, page->index);
  372. entry = xas_load(&xas);
  373. if (dax_is_locked(entry)) {
  374. rcu_read_unlock();
  375. wait_entry_unlocked(&xas, entry);
  376. rcu_read_lock();
  377. continue;
  378. }
  379. dax_lock_entry(&xas, entry);
  380. xas_unlock_irq(&xas);
  381. break;
  382. }
  383. rcu_read_unlock();
  384. return (dax_entry_t)entry;
  385. }
  386. void dax_unlock_page(struct page *page, dax_entry_t cookie)
  387. {
  388. struct address_space *mapping = page->mapping;
  389. XA_STATE(xas, &mapping->i_pages, page->index);
  390. if (S_ISCHR(mapping->host->i_mode))
  391. return;
  392. dax_unlock_entry(&xas, (void *)cookie);
  393. }
  394. /*
  395. * Find page cache entry at given index. If it is a DAX entry, return it
  396. * with the entry locked. If the page cache doesn't contain an entry at
  397. * that index, add a locked empty entry.
  398. *
  399. * When requesting an entry with size DAX_PMD, grab_mapping_entry() will
  400. * either return that locked entry or will return VM_FAULT_FALLBACK.
  401. * This will happen if there are any PTE entries within the PMD range
  402. * that we are requesting.
  403. *
  404. * We always favor PTE entries over PMD entries. There isn't a flow where we
  405. * evict PTE entries in order to 'upgrade' them to a PMD entry. A PMD
  406. * insertion will fail if it finds any PTE entries already in the tree, and a
  407. * PTE insertion will cause an existing PMD entry to be unmapped and
  408. * downgraded to PTE entries. This happens for both PMD zero pages as
  409. * well as PMD empty entries.
  410. *
  411. * The exception to this downgrade path is for PMD entries that have
  412. * real storage backing them. We will leave these real PMD entries in
  413. * the tree, and PTE writes will simply dirty the entire PMD entry.
  414. *
  415. * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
  416. * persistent memory the benefit is doubtful. We can add that later if we can
  417. * show it helps.
  418. *
  419. * On error, this function does not return an ERR_PTR. Instead it returns
  420. * a VM_FAULT code, encoded as an xarray internal entry. The ERR_PTR values
  421. * overlap with xarray value entries.
  422. */
  423. static void *grab_mapping_entry(struct xa_state *xas,
  424. struct address_space *mapping, unsigned int order)
  425. {
  426. unsigned long index = xas->xa_index;
  427. bool pmd_downgrade; /* splitting PMD entry into PTE entries? */
  428. void *entry;
  429. retry:
  430. pmd_downgrade = false;
  431. xas_lock_irq(xas);
  432. entry = get_unlocked_entry(xas, order);
  433. if (entry) {
  434. if (dax_is_conflict(entry))
  435. goto fallback;
  436. if (!xa_is_value(entry)) {
  437. xas_set_err(xas, -EIO);
  438. goto out_unlock;
  439. }
  440. if (order == 0) {
  441. if (dax_is_pmd_entry(entry) &&
  442. (dax_is_zero_entry(entry) ||
  443. dax_is_empty_entry(entry))) {
  444. pmd_downgrade = true;
  445. }
  446. }
  447. }
  448. if (pmd_downgrade) {
  449. /*
  450. * Make sure 'entry' remains valid while we drop
  451. * the i_pages lock.
  452. */
  453. dax_lock_entry(xas, entry);
  454. /*
  455. * Besides huge zero pages the only other thing that gets
  456. * downgraded are empty entries which don't need to be
  457. * unmapped.
  458. */
  459. if (dax_is_zero_entry(entry)) {
  460. xas_unlock_irq(xas);
  461. unmap_mapping_pages(mapping,
  462. xas->xa_index & ~PG_PMD_COLOUR,
  463. PG_PMD_NR, false);
  464. xas_reset(xas);
  465. xas_lock_irq(xas);
  466. }
  467. dax_disassociate_entry(entry, mapping, false);
  468. xas_store(xas, NULL); /* undo the PMD join */
  469. dax_wake_entry(xas, entry, WAKE_ALL);
  470. mapping->nrexceptional--;
  471. entry = NULL;
  472. xas_set(xas, index);
  473. }
  474. if (entry) {
  475. dax_lock_entry(xas, entry);
  476. } else {
  477. unsigned long flags = DAX_EMPTY;
  478. if (order > 0)
  479. flags |= DAX_PMD;
  480. entry = dax_make_entry(pfn_to_pfn_t(0), flags);
  481. dax_lock_entry(xas, entry);
  482. if (xas_error(xas))
  483. goto out_unlock;
  484. mapping->nrexceptional++;
  485. }
  486. out_unlock:
  487. xas_unlock_irq(xas);
  488. if (xas_nomem(xas, mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM))
  489. goto retry;
  490. if (xas->xa_node == XA_ERROR(-ENOMEM))
  491. return xa_mk_internal(VM_FAULT_OOM);
  492. if (xas_error(xas))
  493. return xa_mk_internal(VM_FAULT_SIGBUS);
  494. return entry;
  495. fallback:
  496. xas_unlock_irq(xas);
  497. return xa_mk_internal(VM_FAULT_FALLBACK);
  498. }
  499. /**
  500. * dax_layout_busy_page_range - find first pinned page in @mapping
  501. * @mapping: address space to scan for a page with ref count > 1
  502. * @start: Starting offset. Page containing 'start' is included.
  503. * @end: End offset. Page containing 'end' is included. If 'end' is LLONG_MAX,
  504. * pages from 'start' till the end of file are included.
  505. *
  506. * DAX requires ZONE_DEVICE mapped pages. These pages are never
  507. * 'onlined' to the page allocator so they are considered idle when
  508. * page->count == 1. A filesystem uses this interface to determine if
  509. * any page in the mapping is busy, i.e. for DMA, or other
  510. * get_user_pages() usages.
  511. *
  512. * It is expected that the filesystem is holding locks to block the
  513. * establishment of new mappings in this address_space. I.e. it expects
  514. * to be able to run unmap_mapping_range() and subsequently not race
  515. * mapping_mapped() becoming true.
  516. */
  517. struct page *dax_layout_busy_page_range(struct address_space *mapping,
  518. loff_t start, loff_t end)
  519. {
  520. void *entry;
  521. unsigned int scanned = 0;
  522. struct page *page = NULL;
  523. pgoff_t start_idx = start >> PAGE_SHIFT;
  524. pgoff_t end_idx;
  525. XA_STATE(xas, &mapping->i_pages, start_idx);
  526. /*
  527. * In the 'limited' case get_user_pages() for dax is disabled.
  528. */
  529. if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
  530. return NULL;
  531. if (!dax_mapping(mapping) || !mapping_mapped(mapping))
  532. return NULL;
  533. /* If end == LLONG_MAX, all pages from start to till end of file */
  534. if (end == LLONG_MAX)
  535. end_idx = ULONG_MAX;
  536. else
  537. end_idx = end >> PAGE_SHIFT;
  538. /*
  539. * If we race get_user_pages_fast() here either we'll see the
  540. * elevated page count in the iteration and wait, or
  541. * get_user_pages_fast() will see that the page it took a reference
  542. * against is no longer mapped in the page tables and bail to the
  543. * get_user_pages() slow path. The slow path is protected by
  544. * pte_lock() and pmd_lock(). New references are not taken without
  545. * holding those locks, and unmap_mapping_pages() will not zero the
  546. * pte or pmd without holding the respective lock, so we are
  547. * guaranteed to either see new references or prevent new
  548. * references from being established.
  549. */
  550. unmap_mapping_pages(mapping, start_idx, end_idx - start_idx + 1, 0);
  551. xas_lock_irq(&xas);
  552. xas_for_each(&xas, entry, end_idx) {
  553. if (WARN_ON_ONCE(!xa_is_value(entry)))
  554. continue;
  555. if (unlikely(dax_is_locked(entry)))
  556. entry = get_unlocked_entry(&xas, 0);
  557. if (entry)
  558. page = dax_busy_page(entry);
  559. put_unlocked_entry(&xas, entry, WAKE_NEXT);
  560. if (page)
  561. break;
  562. if (++scanned % XA_CHECK_SCHED)
  563. continue;
  564. xas_pause(&xas);
  565. xas_unlock_irq(&xas);
  566. cond_resched();
  567. xas_lock_irq(&xas);
  568. }
  569. xas_unlock_irq(&xas);
  570. return page;
  571. }
  572. EXPORT_SYMBOL_GPL(dax_layout_busy_page_range);
  573. struct page *dax_layout_busy_page(struct address_space *mapping)
  574. {
  575. return dax_layout_busy_page_range(mapping, 0, LLONG_MAX);
  576. }
  577. EXPORT_SYMBOL_GPL(dax_layout_busy_page);
  578. static int __dax_invalidate_entry(struct address_space *mapping,
  579. pgoff_t index, bool trunc)
  580. {
  581. XA_STATE(xas, &mapping->i_pages, index);
  582. int ret = 0;
  583. void *entry;
  584. xas_lock_irq(&xas);
  585. entry = get_unlocked_entry(&xas, 0);
  586. if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
  587. goto out;
  588. if (!trunc &&
  589. (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY) ||
  590. xas_get_mark(&xas, PAGECACHE_TAG_TOWRITE)))
  591. goto out;
  592. dax_disassociate_entry(entry, mapping, trunc);
  593. xas_store(&xas, NULL);
  594. mapping->nrexceptional--;
  595. ret = 1;
  596. out:
  597. put_unlocked_entry(&xas, entry, WAKE_ALL);
  598. xas_unlock_irq(&xas);
  599. return ret;
  600. }
  601. /*
  602. * Delete DAX entry at @index from @mapping. Wait for it
  603. * to be unlocked before deleting it.
  604. */
  605. int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
  606. {
  607. int ret = __dax_invalidate_entry(mapping, index, true);
  608. /*
  609. * This gets called from truncate / punch_hole path. As such, the caller
  610. * must hold locks protecting against concurrent modifications of the
  611. * page cache (usually fs-private i_mmap_sem for writing). Since the
  612. * caller has seen a DAX entry for this index, we better find it
  613. * at that index as well...
  614. */
  615. WARN_ON_ONCE(!ret);
  616. return ret;
  617. }
  618. /*
  619. * Invalidate DAX entry if it is clean.
  620. */
  621. int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
  622. pgoff_t index)
  623. {
  624. return __dax_invalidate_entry(mapping, index, false);
  625. }
  626. static int copy_cow_page_dax(struct block_device *bdev, struct dax_device *dax_dev,
  627. sector_t sector, struct page *to, unsigned long vaddr)
  628. {
  629. void *vto, *kaddr;
  630. pgoff_t pgoff;
  631. long rc;
  632. int id;
  633. rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
  634. if (rc)
  635. return rc;
  636. id = dax_read_lock();
  637. rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(PAGE_SIZE), &kaddr, NULL);
  638. if (rc < 0) {
  639. dax_read_unlock(id);
  640. return rc;
  641. }
  642. vto = kmap_atomic(to);
  643. copy_user_page(vto, (void __force *)kaddr, vaddr, to);
  644. kunmap_atomic(vto);
  645. dax_read_unlock(id);
  646. return 0;
  647. }
  648. /*
  649. * By this point grab_mapping_entry() has ensured that we have a locked entry
  650. * of the appropriate size so we don't have to worry about downgrading PMDs to
  651. * PTEs. If we happen to be trying to insert a PTE and there is a PMD
  652. * already in the tree, we will skip the insertion and just dirty the PMD as
  653. * appropriate.
  654. */
  655. static void *dax_insert_entry(struct xa_state *xas,
  656. struct address_space *mapping, struct vm_fault *vmf,
  657. void *entry, pfn_t pfn, unsigned long flags, bool dirty)
  658. {
  659. void *new_entry = dax_make_entry(pfn, flags);
  660. if (dirty)
  661. __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
  662. if (dax_is_zero_entry(entry) && !(flags & DAX_ZERO_PAGE)) {
  663. unsigned long index = xas->xa_index;
  664. /* we are replacing a zero page with block mapping */
  665. if (dax_is_pmd_entry(entry))
  666. unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
  667. PG_PMD_NR, false);
  668. else /* pte entry */
  669. unmap_mapping_pages(mapping, index, 1, false);
  670. }
  671. xas_reset(xas);
  672. xas_lock_irq(xas);
  673. if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
  674. void *old;
  675. dax_disassociate_entry(entry, mapping, false);
  676. dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address);
  677. /*
  678. * Only swap our new entry into the page cache if the current
  679. * entry is a zero page or an empty entry. If a normal PTE or
  680. * PMD entry is already in the cache, we leave it alone. This
  681. * means that if we are trying to insert a PTE and the
  682. * existing entry is a PMD, we will just leave the PMD in the
  683. * tree and dirty it if necessary.
  684. */
  685. old = dax_lock_entry(xas, new_entry);
  686. WARN_ON_ONCE(old != xa_mk_value(xa_to_value(entry) |
  687. DAX_LOCKED));
  688. entry = new_entry;
  689. } else {
  690. xas_load(xas); /* Walk the xa_state */
  691. }
  692. if (dirty)
  693. xas_set_mark(xas, PAGECACHE_TAG_DIRTY);
  694. xas_unlock_irq(xas);
  695. return entry;
  696. }
  697. static inline
  698. unsigned long pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
  699. {
  700. unsigned long address;
  701. address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
  702. VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
  703. return address;
  704. }
  705. /* Walk all mappings of a given index of a file and writeprotect them */
  706. static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index,
  707. unsigned long pfn)
  708. {
  709. struct vm_area_struct *vma;
  710. pte_t pte, *ptep = NULL;
  711. pmd_t *pmdp = NULL;
  712. spinlock_t *ptl;
  713. i_mmap_lock_read(mapping);
  714. vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
  715. struct mmu_notifier_range range;
  716. unsigned long address;
  717. cond_resched();
  718. if (!(vma->vm_flags & VM_SHARED))
  719. continue;
  720. address = pgoff_address(index, vma);
  721. /*
  722. * follow_invalidate_pte() will use the range to call
  723. * mmu_notifier_invalidate_range_start() on our behalf before
  724. * taking any lock.
  725. */
  726. if (follow_invalidate_pte(vma->vm_mm, address, &range, &ptep,
  727. &pmdp, &ptl))
  728. continue;
  729. /*
  730. * No need to call mmu_notifier_invalidate_range() as we are
  731. * downgrading page table protection not changing it to point
  732. * to a new page.
  733. *
  734. * See Documentation/vm/mmu_notifier.rst
  735. */
  736. if (pmdp) {
  737. #ifdef CONFIG_FS_DAX_PMD
  738. pmd_t pmd;
  739. if (pfn != pmd_pfn(*pmdp))
  740. goto unlock_pmd;
  741. if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
  742. goto unlock_pmd;
  743. flush_cache_page(vma, address, pfn);
  744. pmd = pmdp_invalidate(vma, address, pmdp);
  745. pmd = pmd_wrprotect(pmd);
  746. pmd = pmd_mkclean(pmd);
  747. set_pmd_at(vma->vm_mm, address, pmdp, pmd);
  748. unlock_pmd:
  749. #endif
  750. spin_unlock(ptl);
  751. } else {
  752. if (pfn != pte_pfn(*ptep))
  753. goto unlock_pte;
  754. if (!pte_dirty(*ptep) && !pte_write(*ptep))
  755. goto unlock_pte;
  756. flush_cache_page(vma, address, pfn);
  757. pte = ptep_clear_flush(vma, address, ptep);
  758. pte = pte_wrprotect(pte);
  759. pte = pte_mkclean(pte);
  760. set_pte_at(vma->vm_mm, address, ptep, pte);
  761. unlock_pte:
  762. pte_unmap_unlock(ptep, ptl);
  763. }
  764. mmu_notifier_invalidate_range_end(&range);
  765. }
  766. i_mmap_unlock_read(mapping);
  767. }
  768. static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev,
  769. struct address_space *mapping, void *entry)
  770. {
  771. unsigned long pfn, index, count;
  772. long ret = 0;
  773. /*
  774. * A page got tagged dirty in DAX mapping? Something is seriously
  775. * wrong.
  776. */
  777. if (WARN_ON(!xa_is_value(entry)))
  778. return -EIO;
  779. if (unlikely(dax_is_locked(entry))) {
  780. void *old_entry = entry;
  781. entry = get_unlocked_entry(xas, 0);
  782. /* Entry got punched out / reallocated? */
  783. if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
  784. goto put_unlocked;
  785. /*
  786. * Entry got reallocated elsewhere? No need to writeback.
  787. * We have to compare pfns as we must not bail out due to
  788. * difference in lockbit or entry type.
  789. */
  790. if (dax_to_pfn(old_entry) != dax_to_pfn(entry))
  791. goto put_unlocked;
  792. if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
  793. dax_is_zero_entry(entry))) {
  794. ret = -EIO;
  795. goto put_unlocked;
  796. }
  797. /* Another fsync thread may have already done this entry */
  798. if (!xas_get_mark(xas, PAGECACHE_TAG_TOWRITE))
  799. goto put_unlocked;
  800. }
  801. /* Lock the entry to serialize with page faults */
  802. dax_lock_entry(xas, entry);
  803. /*
  804. * We can clear the tag now but we have to be careful so that concurrent
  805. * dax_writeback_one() calls for the same index cannot finish before we
  806. * actually flush the caches. This is achieved as the calls will look
  807. * at the entry only under the i_pages lock and once they do that
  808. * they will see the entry locked and wait for it to unlock.
  809. */
  810. xas_clear_mark(xas, PAGECACHE_TAG_TOWRITE);
  811. xas_unlock_irq(xas);
  812. /*
  813. * If dax_writeback_mapping_range() was given a wbc->range_start
  814. * in the middle of a PMD, the 'index' we use needs to be
  815. * aligned to the start of the PMD.
  816. * This allows us to flush for PMD_SIZE and not have to worry about
  817. * partial PMD writebacks.
  818. */
  819. pfn = dax_to_pfn(entry);
  820. count = 1UL << dax_entry_order(entry);
  821. index = xas->xa_index & ~(count - 1);
  822. dax_entry_mkclean(mapping, index, pfn);
  823. dax_flush(dax_dev, page_address(pfn_to_page(pfn)), count * PAGE_SIZE);
  824. /*
  825. * After we have flushed the cache, we can clear the dirty tag. There
  826. * cannot be new dirty data in the pfn after the flush has completed as
  827. * the pfn mappings are writeprotected and fault waits for mapping
  828. * entry lock.
  829. */
  830. xas_reset(xas);
  831. xas_lock_irq(xas);
  832. xas_store(xas, entry);
  833. xas_clear_mark(xas, PAGECACHE_TAG_DIRTY);
  834. dax_wake_entry(xas, entry, WAKE_NEXT);
  835. trace_dax_writeback_one(mapping->host, index, count);
  836. return ret;
  837. put_unlocked:
  838. put_unlocked_entry(xas, entry, WAKE_NEXT);
  839. return ret;
  840. }
  841. /*
  842. * Flush the mapping to the persistent domain within the byte range of [start,
  843. * end]. This is required by data integrity operations to ensure file data is
  844. * on persistent storage prior to completion of the operation.
  845. */
  846. int dax_writeback_mapping_range(struct address_space *mapping,
  847. struct dax_device *dax_dev, struct writeback_control *wbc)
  848. {
  849. XA_STATE(xas, &mapping->i_pages, wbc->range_start >> PAGE_SHIFT);
  850. struct inode *inode = mapping->host;
  851. pgoff_t end_index = wbc->range_end >> PAGE_SHIFT;
  852. void *entry;
  853. int ret = 0;
  854. unsigned int scanned = 0;
  855. if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
  856. return -EIO;
  857. if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
  858. return 0;
  859. trace_dax_writeback_range(inode, xas.xa_index, end_index);
  860. tag_pages_for_writeback(mapping, xas.xa_index, end_index);
  861. xas_lock_irq(&xas);
  862. xas_for_each_marked(&xas, entry, end_index, PAGECACHE_TAG_TOWRITE) {
  863. ret = dax_writeback_one(&xas, dax_dev, mapping, entry);
  864. if (ret < 0) {
  865. mapping_set_error(mapping, ret);
  866. break;
  867. }
  868. if (++scanned % XA_CHECK_SCHED)
  869. continue;
  870. xas_pause(&xas);
  871. xas_unlock_irq(&xas);
  872. cond_resched();
  873. xas_lock_irq(&xas);
  874. }
  875. xas_unlock_irq(&xas);
  876. trace_dax_writeback_range_done(inode, xas.xa_index, end_index);
  877. return ret;
  878. }
  879. EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
  880. static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
  881. {
  882. return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9;
  883. }
  884. static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size,
  885. pfn_t *pfnp)
  886. {
  887. const sector_t sector = dax_iomap_sector(iomap, pos);
  888. pgoff_t pgoff;
  889. int id, rc;
  890. long length;
  891. rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff);
  892. if (rc)
  893. return rc;
  894. id = dax_read_lock();
  895. length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
  896. NULL, pfnp);
  897. if (length < 0) {
  898. rc = length;
  899. goto out;
  900. }
  901. rc = -EINVAL;
  902. if (PFN_PHYS(length) < size)
  903. goto out;
  904. if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1))
  905. goto out;
  906. /* For larger pages we need devmap */
  907. if (length > 1 && !pfn_t_devmap(*pfnp))
  908. goto out;
  909. rc = 0;
  910. out:
  911. dax_read_unlock(id);
  912. return rc;
  913. }
  914. /*
  915. * The user has performed a load from a hole in the file. Allocating a new
  916. * page in the file would cause excessive storage usage for workloads with
  917. * sparse files. Instead we insert a read-only mapping of the 4k zero page.
  918. * If this page is ever written to we will re-fault and change the mapping to
  919. * point to real DAX storage instead.
  920. */
  921. static vm_fault_t dax_load_hole(struct xa_state *xas,
  922. struct address_space *mapping, void **entry,
  923. struct vm_fault *vmf)
  924. {
  925. struct inode *inode = mapping->host;
  926. unsigned long vaddr = vmf->address;
  927. pfn_t pfn = pfn_to_pfn_t(my_zero_pfn(vaddr));
  928. vm_fault_t ret;
  929. *entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,
  930. DAX_ZERO_PAGE, false);
  931. ret = vmf_insert_mixed(vmf->vma, vaddr, pfn);
  932. trace_dax_load_hole(inode, vmf, ret);
  933. return ret;
  934. }
  935. s64 dax_iomap_zero(loff_t pos, u64 length, struct iomap *iomap)
  936. {
  937. sector_t sector = iomap_sector(iomap, pos & PAGE_MASK);
  938. pgoff_t pgoff;
  939. long rc, id;
  940. void *kaddr;
  941. bool page_aligned = false;
  942. unsigned offset = offset_in_page(pos);
  943. unsigned size = min_t(u64, PAGE_SIZE - offset, length);
  944. if (IS_ALIGNED(sector << SECTOR_SHIFT, PAGE_SIZE) &&
  945. (size == PAGE_SIZE))
  946. page_aligned = true;
  947. rc = bdev_dax_pgoff(iomap->bdev, sector, PAGE_SIZE, &pgoff);
  948. if (rc)
  949. return rc;
  950. id = dax_read_lock();
  951. if (page_aligned)
  952. rc = dax_zero_page_range(iomap->dax_dev, pgoff, 1);
  953. else
  954. rc = dax_direct_access(iomap->dax_dev, pgoff, 1, &kaddr, NULL);
  955. if (rc < 0) {
  956. dax_read_unlock(id);
  957. return rc;
  958. }
  959. if (!page_aligned) {
  960. memset(kaddr + offset, 0, size);
  961. dax_flush(iomap->dax_dev, kaddr + offset, size);
  962. }
  963. dax_read_unlock(id);
  964. return size;
  965. }
  966. static loff_t
  967. dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
  968. struct iomap *iomap, struct iomap *srcmap)
  969. {
  970. struct block_device *bdev = iomap->bdev;
  971. struct dax_device *dax_dev = iomap->dax_dev;
  972. struct iov_iter *iter = data;
  973. loff_t end = pos + length, done = 0;
  974. ssize_t ret = 0;
  975. size_t xfer;
  976. int id;
  977. if (iov_iter_rw(iter) == READ) {
  978. end = min(end, i_size_read(inode));
  979. if (pos >= end)
  980. return 0;
  981. if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
  982. return iov_iter_zero(min(length, end - pos), iter);
  983. }
  984. if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
  985. return -EIO;
  986. /*
  987. * Write can allocate block for an area which has a hole page mapped
  988. * into page tables. We have to tear down these mappings so that data
  989. * written by write(2) is visible in mmap.
  990. */
  991. if (iomap->flags & IOMAP_F_NEW) {
  992. invalidate_inode_pages2_range(inode->i_mapping,
  993. pos >> PAGE_SHIFT,
  994. (end - 1) >> PAGE_SHIFT);
  995. }
  996. id = dax_read_lock();
  997. while (pos < end) {
  998. unsigned offset = pos & (PAGE_SIZE - 1);
  999. const size_t size = ALIGN(length + offset, PAGE_SIZE);
  1000. const sector_t sector = dax_iomap_sector(iomap, pos);
  1001. ssize_t map_len;
  1002. pgoff_t pgoff;
  1003. void *kaddr;
  1004. if (fatal_signal_pending(current)) {
  1005. ret = -EINTR;
  1006. break;
  1007. }
  1008. ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
  1009. if (ret)
  1010. break;
  1011. map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
  1012. &kaddr, NULL);
  1013. if (map_len < 0) {
  1014. ret = map_len;
  1015. break;
  1016. }
  1017. map_len = PFN_PHYS(map_len);
  1018. kaddr += offset;
  1019. map_len -= offset;
  1020. if (map_len > end - pos)
  1021. map_len = end - pos;
  1022. /*
  1023. * The userspace address for the memory copy has already been
  1024. * validated via access_ok() in either vfs_read() or
  1025. * vfs_write(), depending on which operation we are doing.
  1026. */
  1027. if (iov_iter_rw(iter) == WRITE)
  1028. xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr,
  1029. map_len, iter);
  1030. else
  1031. xfer = dax_copy_to_iter(dax_dev, pgoff, kaddr,
  1032. map_len, iter);
  1033. pos += xfer;
  1034. length -= xfer;
  1035. done += xfer;
  1036. if (xfer == 0)
  1037. ret = -EFAULT;
  1038. if (xfer < map_len)
  1039. break;
  1040. }
  1041. dax_read_unlock(id);
  1042. return done ? done : ret;
  1043. }
  1044. /**
  1045. * dax_iomap_rw - Perform I/O to a DAX file
  1046. * @iocb: The control block for this I/O
  1047. * @iter: The addresses to do I/O from or to
  1048. * @ops: iomap ops passed from the file system
  1049. *
  1050. * This function performs read and write operations to directly mapped
  1051. * persistent memory. The callers needs to take care of read/write exclusion
  1052. * and evicting any page cache pages in the region under I/O.
  1053. */
  1054. ssize_t
  1055. dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
  1056. const struct iomap_ops *ops)
  1057. {
  1058. struct address_space *mapping = iocb->ki_filp->f_mapping;
  1059. struct inode *inode = mapping->host;
  1060. loff_t pos = iocb->ki_pos, ret = 0, done = 0;
  1061. unsigned flags = 0;
  1062. if (iov_iter_rw(iter) == WRITE) {
  1063. lockdep_assert_held_write(&inode->i_rwsem);
  1064. flags |= IOMAP_WRITE;
  1065. } else {
  1066. lockdep_assert_held(&inode->i_rwsem);
  1067. }
  1068. if (iocb->ki_flags & IOCB_NOWAIT)
  1069. flags |= IOMAP_NOWAIT;
  1070. while (iov_iter_count(iter)) {
  1071. ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
  1072. iter, dax_iomap_actor);
  1073. if (ret <= 0)
  1074. break;
  1075. pos += ret;
  1076. done += ret;
  1077. }
  1078. iocb->ki_pos += done;
  1079. return done ? done : ret;
  1080. }
  1081. EXPORT_SYMBOL_GPL(dax_iomap_rw);
  1082. static vm_fault_t dax_fault_return(int error)
  1083. {
  1084. if (error == 0)
  1085. return VM_FAULT_NOPAGE;
  1086. return vmf_error(error);
  1087. }
  1088. /*
  1089. * MAP_SYNC on a dax mapping guarantees dirty metadata is
  1090. * flushed on write-faults (non-cow), but not read-faults.
  1091. */
  1092. static bool dax_fault_is_synchronous(unsigned long flags,
  1093. struct vm_area_struct *vma, struct iomap *iomap)
  1094. {
  1095. return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC)
  1096. && (iomap->flags & IOMAP_F_DIRTY);
  1097. }
  1098. static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
  1099. int *iomap_errp, const struct iomap_ops *ops)
  1100. {
  1101. struct vm_area_struct *vma = vmf->vma;
  1102. struct address_space *mapping = vma->vm_file->f_mapping;
  1103. XA_STATE(xas, &mapping->i_pages, vmf->pgoff);
  1104. struct inode *inode = mapping->host;
  1105. unsigned long vaddr = vmf->address;
  1106. loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
  1107. struct iomap iomap = { .type = IOMAP_HOLE };
  1108. struct iomap srcmap = { .type = IOMAP_HOLE };
  1109. unsigned flags = IOMAP_FAULT;
  1110. int error, major = 0;
  1111. bool write = vmf->flags & FAULT_FLAG_WRITE;
  1112. bool sync;
  1113. vm_fault_t ret = 0;
  1114. void *entry;
  1115. pfn_t pfn;
  1116. trace_dax_pte_fault(inode, vmf, ret);
  1117. /*
  1118. * Check whether offset isn't beyond end of file now. Caller is supposed
  1119. * to hold locks serializing us with truncate / punch hole so this is
  1120. * a reliable test.
  1121. */
  1122. if (pos >= i_size_read(inode)) {
  1123. ret = VM_FAULT_SIGBUS;
  1124. goto out;
  1125. }
  1126. if (write && !vmf->cow_page)
  1127. flags |= IOMAP_WRITE;
  1128. entry = grab_mapping_entry(&xas, mapping, 0);
  1129. if (xa_is_internal(entry)) {
  1130. ret = xa_to_internal(entry);
  1131. goto out;
  1132. }
  1133. /*
  1134. * It is possible, particularly with mixed reads & writes to private
  1135. * mappings, that we have raced with a PMD fault that overlaps with
  1136. * the PTE we need to set up. If so just return and the fault will be
  1137. * retried.
  1138. */
  1139. if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {
  1140. ret = VM_FAULT_NOPAGE;
  1141. goto unlock_entry;
  1142. }
  1143. /*
  1144. * Note that we don't bother to use iomap_apply here: DAX required
  1145. * the file system block size to be equal the page size, which means
  1146. * that we never have to deal with more than a single extent here.
  1147. */
  1148. error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap, &srcmap);
  1149. if (iomap_errp)
  1150. *iomap_errp = error;
  1151. if (error) {
  1152. ret = dax_fault_return(error);
  1153. goto unlock_entry;
  1154. }
  1155. if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
  1156. error = -EIO; /* fs corruption? */
  1157. goto error_finish_iomap;
  1158. }
  1159. if (vmf->cow_page) {
  1160. sector_t sector = dax_iomap_sector(&iomap, pos);
  1161. switch (iomap.type) {
  1162. case IOMAP_HOLE:
  1163. case IOMAP_UNWRITTEN:
  1164. clear_user_highpage(vmf->cow_page, vaddr);
  1165. break;
  1166. case IOMAP_MAPPED:
  1167. error = copy_cow_page_dax(iomap.bdev, iomap.dax_dev,
  1168. sector, vmf->cow_page, vaddr);
  1169. break;
  1170. default:
  1171. WARN_ON_ONCE(1);
  1172. error = -EIO;
  1173. break;
  1174. }
  1175. if (error)
  1176. goto error_finish_iomap;
  1177. __SetPageUptodate(vmf->cow_page);
  1178. ret = finish_fault(vmf);
  1179. if (!ret)
  1180. ret = VM_FAULT_DONE_COW;
  1181. goto finish_iomap;
  1182. }
  1183. sync = dax_fault_is_synchronous(flags, vma, &iomap);
  1184. switch (iomap.type) {
  1185. case IOMAP_MAPPED:
  1186. if (iomap.flags & IOMAP_F_NEW) {
  1187. count_vm_event(PGMAJFAULT);
  1188. count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
  1189. major = VM_FAULT_MAJOR;
  1190. }
  1191. error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn);
  1192. if (error < 0)
  1193. goto error_finish_iomap;
  1194. entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,
  1195. 0, write && !sync);
  1196. /*
  1197. * If we are doing synchronous page fault and inode needs fsync,
  1198. * we can insert PTE into page tables only after that happens.
  1199. * Skip insertion for now and return the pfn so that caller can
  1200. * insert it after fsync is done.
  1201. */
  1202. if (sync) {
  1203. if (WARN_ON_ONCE(!pfnp)) {
  1204. error = -EIO;
  1205. goto error_finish_iomap;
  1206. }
  1207. *pfnp = pfn;
  1208. ret = VM_FAULT_NEEDDSYNC | major;
  1209. goto finish_iomap;
  1210. }
  1211. trace_dax_insert_mapping(inode, vmf, entry);
  1212. if (write)
  1213. ret = vmf_insert_mixed_mkwrite(vma, vaddr, pfn);
  1214. else
  1215. ret = vmf_insert_mixed(vma, vaddr, pfn);
  1216. goto finish_iomap;
  1217. case IOMAP_UNWRITTEN:
  1218. case IOMAP_HOLE:
  1219. if (!write) {
  1220. ret = dax_load_hole(&xas, mapping, &entry, vmf);
  1221. goto finish_iomap;
  1222. }
  1223. fallthrough;
  1224. default:
  1225. WARN_ON_ONCE(1);
  1226. error = -EIO;
  1227. break;
  1228. }
  1229. error_finish_iomap:
  1230. ret = dax_fault_return(error);
  1231. finish_iomap:
  1232. if (ops->iomap_end) {
  1233. int copied = PAGE_SIZE;
  1234. if (ret & VM_FAULT_ERROR)
  1235. copied = 0;
  1236. /*
  1237. * The fault is done by now and there's no way back (other
  1238. * thread may be already happily using PTE we have installed).
  1239. * Just ignore error from ->iomap_end since we cannot do much
  1240. * with it.
  1241. */
  1242. ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
  1243. }
  1244. unlock_entry:
  1245. dax_unlock_entry(&xas, entry);
  1246. out:
  1247. trace_dax_pte_fault_done(inode, vmf, ret);
  1248. return ret | major;
  1249. }
  1250. #ifdef CONFIG_FS_DAX_PMD
  1251. static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,
  1252. struct iomap *iomap, void **entry)
  1253. {
  1254. struct address_space *mapping = vmf->vma->vm_file->f_mapping;
  1255. unsigned long pmd_addr = vmf->address & PMD_MASK;
  1256. struct vm_area_struct *vma = vmf->vma;
  1257. struct inode *inode = mapping->host;
  1258. pgtable_t pgtable = NULL;
  1259. struct page *zero_page;
  1260. spinlock_t *ptl;
  1261. pmd_t pmd_entry;
  1262. pfn_t pfn;
  1263. zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);
  1264. if (unlikely(!zero_page))
  1265. goto fallback;
  1266. pfn = page_to_pfn_t(zero_page);
  1267. *entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,
  1268. DAX_PMD | DAX_ZERO_PAGE, false);
  1269. if (arch_needs_pgtable_deposit()) {
  1270. pgtable = pte_alloc_one(vma->vm_mm);
  1271. if (!pgtable)
  1272. return VM_FAULT_OOM;
  1273. }
  1274. ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
  1275. if (!pmd_none(*(vmf->pmd))) {
  1276. spin_unlock(ptl);
  1277. goto fallback;
  1278. }
  1279. if (pgtable) {
  1280. pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
  1281. mm_inc_nr_ptes(vma->vm_mm);
  1282. }
  1283. pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);
  1284. pmd_entry = pmd_mkhuge(pmd_entry);
  1285. set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);
  1286. spin_unlock(ptl);
  1287. trace_dax_pmd_load_hole(inode, vmf, zero_page, *entry);
  1288. return VM_FAULT_NOPAGE;
  1289. fallback:
  1290. if (pgtable)
  1291. pte_free(vma->vm_mm, pgtable);
  1292. trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, *entry);
  1293. return VM_FAULT_FALLBACK;
  1294. }
  1295. static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
  1296. const struct iomap_ops *ops)
  1297. {
  1298. struct vm_area_struct *vma = vmf->vma;
  1299. struct address_space *mapping = vma->vm_file->f_mapping;
  1300. XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, PMD_ORDER);
  1301. unsigned long pmd_addr = vmf->address & PMD_MASK;
  1302. bool write = vmf->flags & FAULT_FLAG_WRITE;
  1303. bool sync;
  1304. unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
  1305. struct inode *inode = mapping->host;
  1306. vm_fault_t result = VM_FAULT_FALLBACK;
  1307. struct iomap iomap = { .type = IOMAP_HOLE };
  1308. struct iomap srcmap = { .type = IOMAP_HOLE };
  1309. pgoff_t max_pgoff;
  1310. void *entry;
  1311. loff_t pos;
  1312. int error;
  1313. pfn_t pfn;
  1314. /*
  1315. * Check whether offset isn't beyond end of file now. Caller is
  1316. * supposed to hold locks serializing us with truncate / punch hole so
  1317. * this is a reliable test.
  1318. */
  1319. max_pgoff = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
  1320. trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);
  1321. /*
  1322. * Make sure that the faulting address's PMD offset (color) matches
  1323. * the PMD offset from the start of the file. This is necessary so
  1324. * that a PMD range in the page table overlaps exactly with a PMD
  1325. * range in the page cache.
  1326. */
  1327. if ((vmf->pgoff & PG_PMD_COLOUR) !=
  1328. ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))
  1329. goto fallback;
  1330. /* Fall back to PTEs if we're going to COW */
  1331. if (write && !(vma->vm_flags & VM_SHARED))
  1332. goto fallback;
  1333. /* If the PMD would extend outside the VMA */
  1334. if (pmd_addr < vma->vm_start)
  1335. goto fallback;
  1336. if ((pmd_addr + PMD_SIZE) > vma->vm_end)
  1337. goto fallback;
  1338. if (xas.xa_index >= max_pgoff) {
  1339. result = VM_FAULT_SIGBUS;
  1340. goto out;
  1341. }
  1342. /* If the PMD would extend beyond the file size */
  1343. if ((xas.xa_index | PG_PMD_COLOUR) >= max_pgoff)
  1344. goto fallback;
  1345. /*
  1346. * grab_mapping_entry() will make sure we get an empty PMD entry,
  1347. * a zero PMD entry or a DAX PMD. If it can't (because a PTE
  1348. * entry is already in the array, for instance), it will return
  1349. * VM_FAULT_FALLBACK.
  1350. */
  1351. entry = grab_mapping_entry(&xas, mapping, PMD_ORDER);
  1352. if (xa_is_internal(entry)) {
  1353. result = xa_to_internal(entry);
  1354. goto fallback;
  1355. }
  1356. /*
  1357. * It is possible, particularly with mixed reads & writes to private
  1358. * mappings, that we have raced with a PTE fault that overlaps with
  1359. * the PMD we need to set up. If so just return and the fault will be
  1360. * retried.
  1361. */
  1362. if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&
  1363. !pmd_devmap(*vmf->pmd)) {
  1364. result = 0;
  1365. goto unlock_entry;
  1366. }
  1367. /*
  1368. * Note that we don't use iomap_apply here. We aren't doing I/O, only
  1369. * setting up a mapping, so really we're using iomap_begin() as a way
  1370. * to look up our filesystem block.
  1371. */
  1372. pos = (loff_t)xas.xa_index << PAGE_SHIFT;
  1373. error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap,
  1374. &srcmap);
  1375. if (error)
  1376. goto unlock_entry;
  1377. if (iomap.offset + iomap.length < pos + PMD_SIZE)
  1378. goto finish_iomap;
  1379. sync = dax_fault_is_synchronous(iomap_flags, vma, &iomap);
  1380. switch (iomap.type) {
  1381. case IOMAP_MAPPED:
  1382. error = dax_iomap_pfn(&iomap, pos, PMD_SIZE, &pfn);
  1383. if (error < 0)
  1384. goto finish_iomap;
  1385. entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,
  1386. DAX_PMD, write && !sync);
  1387. /*
  1388. * If we are doing synchronous page fault and inode needs fsync,
  1389. * we can insert PMD into page tables only after that happens.
  1390. * Skip insertion for now and return the pfn so that caller can
  1391. * insert it after fsync is done.
  1392. */
  1393. if (sync) {
  1394. if (WARN_ON_ONCE(!pfnp))
  1395. goto finish_iomap;
  1396. *pfnp = pfn;
  1397. result = VM_FAULT_NEEDDSYNC;
  1398. goto finish_iomap;
  1399. }
  1400. trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry);
  1401. result = vmf_insert_pfn_pmd(vmf, pfn, write);
  1402. break;
  1403. case IOMAP_UNWRITTEN:
  1404. case IOMAP_HOLE:
  1405. if (WARN_ON_ONCE(write))
  1406. break;
  1407. result = dax_pmd_load_hole(&xas, vmf, &iomap, &entry);
  1408. break;
  1409. default:
  1410. WARN_ON_ONCE(1);
  1411. break;
  1412. }
  1413. finish_iomap:
  1414. if (ops->iomap_end) {
  1415. int copied = PMD_SIZE;
  1416. if (result == VM_FAULT_FALLBACK)
  1417. copied = 0;
  1418. /*
  1419. * The fault is done by now and there's no way back (other
  1420. * thread may be already happily using PMD we have installed).
  1421. * Just ignore error from ->iomap_end since we cannot do much
  1422. * with it.
  1423. */
  1424. ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
  1425. &iomap);
  1426. }
  1427. unlock_entry:
  1428. dax_unlock_entry(&xas, entry);
  1429. fallback:
  1430. if (result == VM_FAULT_FALLBACK) {
  1431. split_huge_pmd(vma, vmf->pmd, vmf->address);
  1432. count_vm_event(THP_FAULT_FALLBACK);
  1433. }
  1434. out:
  1435. trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result);
  1436. return result;
  1437. }
  1438. #else
  1439. static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
  1440. const struct iomap_ops *ops)
  1441. {
  1442. return VM_FAULT_FALLBACK;
  1443. }
  1444. #endif /* CONFIG_FS_DAX_PMD */
  1445. /**
  1446. * dax_iomap_fault - handle a page fault on a DAX file
  1447. * @vmf: The description of the fault
  1448. * @pe_size: Size of the page to fault in
  1449. * @pfnp: PFN to insert for synchronous faults if fsync is required
  1450. * @iomap_errp: Storage for detailed error code in case of error
  1451. * @ops: Iomap ops passed from the file system
  1452. *
  1453. * When a page fault occurs, filesystems may call this helper in
  1454. * their fault handler for DAX files. dax_iomap_fault() assumes the caller
  1455. * has done all the necessary locking for page fault to proceed
  1456. * successfully.
  1457. */
  1458. vm_fault_t dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
  1459. pfn_t *pfnp, int *iomap_errp, const struct iomap_ops *ops)
  1460. {
  1461. switch (pe_size) {
  1462. case PE_SIZE_PTE:
  1463. return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops);
  1464. case PE_SIZE_PMD:
  1465. return dax_iomap_pmd_fault(vmf, pfnp, ops);
  1466. default:
  1467. return VM_FAULT_FALLBACK;
  1468. }
  1469. }
  1470. EXPORT_SYMBOL_GPL(dax_iomap_fault);
  1471. /*
  1472. * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables
  1473. * @vmf: The description of the fault
  1474. * @pfn: PFN to insert
  1475. * @order: Order of entry to insert.
  1476. *
  1477. * This function inserts a writeable PTE or PMD entry into the page tables
  1478. * for an mmaped DAX file. It also marks the page cache entry as dirty.
  1479. */
  1480. static vm_fault_t
  1481. dax_insert_pfn_mkwrite(struct vm_fault *vmf, pfn_t pfn, unsigned int order)
  1482. {
  1483. struct address_space *mapping = vmf->vma->vm_file->f_mapping;
  1484. XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, order);
  1485. void *entry;
  1486. vm_fault_t ret;
  1487. xas_lock_irq(&xas);
  1488. entry = get_unlocked_entry(&xas, order);
  1489. /* Did we race with someone splitting entry or so? */
  1490. if (!entry || dax_is_conflict(entry) ||
  1491. (order == 0 && !dax_is_pte_entry(entry))) {
  1492. put_unlocked_entry(&xas, entry, WAKE_NEXT);
  1493. xas_unlock_irq(&xas);
  1494. trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,
  1495. VM_FAULT_NOPAGE);
  1496. return VM_FAULT_NOPAGE;
  1497. }
  1498. xas_set_mark(&xas, PAGECACHE_TAG_DIRTY);
  1499. dax_lock_entry(&xas, entry);
  1500. xas_unlock_irq(&xas);
  1501. if (order == 0)
  1502. ret = vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);
  1503. #ifdef CONFIG_FS_DAX_PMD
  1504. else if (order == PMD_ORDER)
  1505. ret = vmf_insert_pfn_pmd(vmf, pfn, FAULT_FLAG_WRITE);
  1506. #endif
  1507. else
  1508. ret = VM_FAULT_FALLBACK;
  1509. dax_unlock_entry(&xas, entry);
  1510. trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret);
  1511. return ret;
  1512. }
  1513. /**
  1514. * dax_finish_sync_fault - finish synchronous page fault
  1515. * @vmf: The description of the fault
  1516. * @pe_size: Size of entry to be inserted
  1517. * @pfn: PFN to insert
  1518. *
  1519. * This function ensures that the file range touched by the page fault is
  1520. * stored persistently on the media and handles inserting of appropriate page
  1521. * table entry.
  1522. */
  1523. vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
  1524. enum page_entry_size pe_size, pfn_t pfn)
  1525. {
  1526. int err;
  1527. loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
  1528. unsigned int order = pe_order(pe_size);
  1529. size_t len = PAGE_SIZE << order;
  1530. err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);
  1531. if (err)
  1532. return VM_FAULT_SIGBUS;
  1533. return dax_insert_pfn_mkwrite(vmf, pfn, order);
  1534. }
  1535. EXPORT_SYMBOL_GPL(dax_finish_sync_fault);