xfs_file.c 36 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  4. * All Rights Reserved.
  5. */
  6. #include "xfs.h"
  7. #include "xfs_fs.h"
  8. #include "xfs_shared.h"
  9. #include "xfs_format.h"
  10. #include "xfs_log_format.h"
  11. #include "xfs_trans_resv.h"
  12. #include "xfs_mount.h"
  13. #include "xfs_inode.h"
  14. #include "xfs_trans.h"
  15. #include "xfs_inode_item.h"
  16. #include "xfs_bmap.h"
  17. #include "xfs_bmap_util.h"
  18. #include "xfs_dir2.h"
  19. #include "xfs_dir2_priv.h"
  20. #include "xfs_ioctl.h"
  21. #include "xfs_trace.h"
  22. #include "xfs_log.h"
  23. #include "xfs_icache.h"
  24. #include "xfs_pnfs.h"
  25. #include "xfs_iomap.h"
  26. #include "xfs_reflink.h"
  27. #include <linux/falloc.h>
  28. #include <linux/backing-dev.h>
  29. #include <linux/mman.h>
  30. #include <linux/fadvise.h>
  31. static const struct vm_operations_struct xfs_file_vm_ops;
  32. /*
  33. * Decide if the given file range is aligned to the size of the fundamental
  34. * allocation unit for the file.
  35. */
  36. static bool
  37. xfs_is_falloc_aligned(
  38. struct xfs_inode *ip,
  39. loff_t pos,
  40. long long int len)
  41. {
  42. struct xfs_mount *mp = ip->i_mount;
  43. uint64_t mask;
  44. if (XFS_IS_REALTIME_INODE(ip)) {
  45. if (!is_power_of_2(mp->m_sb.sb_rextsize)) {
  46. u64 rextbytes;
  47. u32 mod;
  48. rextbytes = XFS_FSB_TO_B(mp, mp->m_sb.sb_rextsize);
  49. div_u64_rem(pos, rextbytes, &mod);
  50. if (mod)
  51. return false;
  52. div_u64_rem(len, rextbytes, &mod);
  53. return mod == 0;
  54. }
  55. mask = XFS_FSB_TO_B(mp, mp->m_sb.sb_rextsize) - 1;
  56. } else {
  57. mask = mp->m_sb.sb_blocksize - 1;
  58. }
  59. return !((pos | len) & mask);
  60. }
  61. int
  62. xfs_update_prealloc_flags(
  63. struct xfs_inode *ip,
  64. enum xfs_prealloc_flags flags)
  65. {
  66. struct xfs_trans *tp;
  67. int error;
  68. error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_writeid,
  69. 0, 0, 0, &tp);
  70. if (error)
  71. return error;
  72. xfs_ilock(ip, XFS_ILOCK_EXCL);
  73. xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
  74. if (!(flags & XFS_PREALLOC_INVISIBLE)) {
  75. VFS_I(ip)->i_mode &= ~S_ISUID;
  76. if (VFS_I(ip)->i_mode & S_IXGRP)
  77. VFS_I(ip)->i_mode &= ~S_ISGID;
  78. xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG);
  79. }
  80. if (flags & XFS_PREALLOC_SET)
  81. ip->i_d.di_flags |= XFS_DIFLAG_PREALLOC;
  82. if (flags & XFS_PREALLOC_CLEAR)
  83. ip->i_d.di_flags &= ~XFS_DIFLAG_PREALLOC;
  84. xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
  85. if (flags & XFS_PREALLOC_SYNC)
  86. xfs_trans_set_sync(tp);
  87. return xfs_trans_commit(tp);
  88. }
  89. /*
  90. * Fsync operations on directories are much simpler than on regular files,
  91. * as there is no file data to flush, and thus also no need for explicit
  92. * cache flush operations, and there are no non-transaction metadata updates
  93. * on directories either.
  94. */
  95. STATIC int
  96. xfs_dir_fsync(
  97. struct file *file,
  98. loff_t start,
  99. loff_t end,
  100. int datasync)
  101. {
  102. struct xfs_inode *ip = XFS_I(file->f_mapping->host);
  103. trace_xfs_dir_fsync(ip);
  104. return xfs_log_force_inode(ip);
  105. }
  106. STATIC int
  107. xfs_file_fsync(
  108. struct file *file,
  109. loff_t start,
  110. loff_t end,
  111. int datasync)
  112. {
  113. struct inode *inode = file->f_mapping->host;
  114. struct xfs_inode *ip = XFS_I(inode);
  115. struct xfs_inode_log_item *iip = ip->i_itemp;
  116. struct xfs_mount *mp = ip->i_mount;
  117. int error = 0;
  118. int log_flushed = 0;
  119. xfs_lsn_t lsn = 0;
  120. trace_xfs_file_fsync(ip);
  121. error = file_write_and_wait_range(file, start, end);
  122. if (error)
  123. return error;
  124. if (XFS_FORCED_SHUTDOWN(mp))
  125. return -EIO;
  126. xfs_iflags_clear(ip, XFS_ITRUNCATED);
  127. /*
  128. * If we have an RT and/or log subvolume we need to make sure to flush
  129. * the write cache the device used for file data first. This is to
  130. * ensure newly written file data make it to disk before logging the new
  131. * inode size in case of an extending write.
  132. */
  133. if (XFS_IS_REALTIME_INODE(ip))
  134. xfs_blkdev_issue_flush(mp->m_rtdev_targp);
  135. else if (mp->m_logdev_targp != mp->m_ddev_targp)
  136. xfs_blkdev_issue_flush(mp->m_ddev_targp);
  137. /*
  138. * All metadata updates are logged, which means that we just have to
  139. * flush the log up to the latest LSN that touched the inode. If we have
  140. * concurrent fsync/fdatasync() calls, we need them to all block on the
  141. * log force before we clear the ili_fsync_fields field. This ensures
  142. * that we don't get a racing sync operation that does not wait for the
  143. * metadata to hit the journal before returning. If we race with
  144. * clearing the ili_fsync_fields, then all that will happen is the log
  145. * force will do nothing as the lsn will already be on disk. We can't
  146. * race with setting ili_fsync_fields because that is done under
  147. * XFS_ILOCK_EXCL, and that can't happen because we hold the lock shared
  148. * until after the ili_fsync_fields is cleared.
  149. */
  150. xfs_ilock(ip, XFS_ILOCK_SHARED);
  151. if (xfs_ipincount(ip)) {
  152. if (!datasync ||
  153. (iip->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
  154. lsn = iip->ili_last_lsn;
  155. }
  156. if (lsn) {
  157. error = xfs_log_force_lsn(mp, lsn, XFS_LOG_SYNC, &log_flushed);
  158. spin_lock(&iip->ili_lock);
  159. iip->ili_fsync_fields = 0;
  160. spin_unlock(&iip->ili_lock);
  161. }
  162. xfs_iunlock(ip, XFS_ILOCK_SHARED);
  163. /*
  164. * If we only have a single device, and the log force about was
  165. * a no-op we might have to flush the data device cache here.
  166. * This can only happen for fdatasync/O_DSYNC if we were overwriting
  167. * an already allocated file and thus do not have any metadata to
  168. * commit.
  169. */
  170. if (!log_flushed && !XFS_IS_REALTIME_INODE(ip) &&
  171. mp->m_logdev_targp == mp->m_ddev_targp)
  172. xfs_blkdev_issue_flush(mp->m_ddev_targp);
  173. return error;
  174. }
  175. STATIC ssize_t
  176. xfs_file_dio_aio_read(
  177. struct kiocb *iocb,
  178. struct iov_iter *to)
  179. {
  180. struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp));
  181. size_t count = iov_iter_count(to);
  182. ssize_t ret;
  183. trace_xfs_file_direct_read(ip, count, iocb->ki_pos);
  184. if (!count)
  185. return 0; /* skip atime */
  186. file_accessed(iocb->ki_filp);
  187. if (iocb->ki_flags & IOCB_NOWAIT) {
  188. if (!xfs_ilock_nowait(ip, XFS_IOLOCK_SHARED))
  189. return -EAGAIN;
  190. } else {
  191. xfs_ilock(ip, XFS_IOLOCK_SHARED);
  192. }
  193. ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL,
  194. is_sync_kiocb(iocb));
  195. xfs_iunlock(ip, XFS_IOLOCK_SHARED);
  196. return ret;
  197. }
  198. static noinline ssize_t
  199. xfs_file_dax_read(
  200. struct kiocb *iocb,
  201. struct iov_iter *to)
  202. {
  203. struct xfs_inode *ip = XFS_I(iocb->ki_filp->f_mapping->host);
  204. size_t count = iov_iter_count(to);
  205. ssize_t ret = 0;
  206. trace_xfs_file_dax_read(ip, count, iocb->ki_pos);
  207. if (!count)
  208. return 0; /* skip atime */
  209. if (iocb->ki_flags & IOCB_NOWAIT) {
  210. if (!xfs_ilock_nowait(ip, XFS_IOLOCK_SHARED))
  211. return -EAGAIN;
  212. } else {
  213. xfs_ilock(ip, XFS_IOLOCK_SHARED);
  214. }
  215. ret = dax_iomap_rw(iocb, to, &xfs_read_iomap_ops);
  216. xfs_iunlock(ip, XFS_IOLOCK_SHARED);
  217. file_accessed(iocb->ki_filp);
  218. return ret;
  219. }
  220. STATIC ssize_t
  221. xfs_file_buffered_aio_read(
  222. struct kiocb *iocb,
  223. struct iov_iter *to)
  224. {
  225. struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp));
  226. ssize_t ret;
  227. trace_xfs_file_buffered_read(ip, iov_iter_count(to), iocb->ki_pos);
  228. if (iocb->ki_flags & IOCB_NOWAIT) {
  229. if (!xfs_ilock_nowait(ip, XFS_IOLOCK_SHARED))
  230. return -EAGAIN;
  231. } else {
  232. xfs_ilock(ip, XFS_IOLOCK_SHARED);
  233. }
  234. ret = generic_file_read_iter(iocb, to);
  235. xfs_iunlock(ip, XFS_IOLOCK_SHARED);
  236. return ret;
  237. }
  238. STATIC ssize_t
  239. xfs_file_read_iter(
  240. struct kiocb *iocb,
  241. struct iov_iter *to)
  242. {
  243. struct inode *inode = file_inode(iocb->ki_filp);
  244. struct xfs_mount *mp = XFS_I(inode)->i_mount;
  245. ssize_t ret = 0;
  246. XFS_STATS_INC(mp, xs_read_calls);
  247. if (XFS_FORCED_SHUTDOWN(mp))
  248. return -EIO;
  249. if (IS_DAX(inode))
  250. ret = xfs_file_dax_read(iocb, to);
  251. else if (iocb->ki_flags & IOCB_DIRECT)
  252. ret = xfs_file_dio_aio_read(iocb, to);
  253. else
  254. ret = xfs_file_buffered_aio_read(iocb, to);
  255. if (ret > 0)
  256. XFS_STATS_ADD(mp, xs_read_bytes, ret);
  257. return ret;
  258. }
  259. /*
  260. * Common pre-write limit and setup checks.
  261. *
  262. * Called with the iolocked held either shared and exclusive according to
  263. * @iolock, and returns with it held. Might upgrade the iolock to exclusive
  264. * if called for a direct write beyond i_size.
  265. */
  266. STATIC ssize_t
  267. xfs_file_aio_write_checks(
  268. struct kiocb *iocb,
  269. struct iov_iter *from,
  270. int *iolock)
  271. {
  272. struct file *file = iocb->ki_filp;
  273. struct inode *inode = file->f_mapping->host;
  274. struct xfs_inode *ip = XFS_I(inode);
  275. ssize_t error = 0;
  276. size_t count = iov_iter_count(from);
  277. bool drained_dio = false;
  278. loff_t isize;
  279. restart:
  280. error = generic_write_checks(iocb, from);
  281. if (error <= 0)
  282. return error;
  283. error = xfs_break_layouts(inode, iolock, BREAK_WRITE);
  284. if (error)
  285. return error;
  286. /*
  287. * For changing security info in file_remove_privs() we need i_rwsem
  288. * exclusively.
  289. */
  290. if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) {
  291. xfs_iunlock(ip, *iolock);
  292. *iolock = XFS_IOLOCK_EXCL;
  293. xfs_ilock(ip, *iolock);
  294. goto restart;
  295. }
  296. /*
  297. * If the offset is beyond the size of the file, we need to zero any
  298. * blocks that fall between the existing EOF and the start of this
  299. * write. If zeroing is needed and we are currently holding the
  300. * iolock shared, we need to update it to exclusive which implies
  301. * having to redo all checks before.
  302. *
  303. * We need to serialise against EOF updates that occur in IO
  304. * completions here. We want to make sure that nobody is changing the
  305. * size while we do this check until we have placed an IO barrier (i.e.
  306. * hold the XFS_IOLOCK_EXCL) that prevents new IO from being dispatched.
  307. * The spinlock effectively forms a memory barrier once we have the
  308. * XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value
  309. * and hence be able to correctly determine if we need to run zeroing.
  310. */
  311. spin_lock(&ip->i_flags_lock);
  312. isize = i_size_read(inode);
  313. if (iocb->ki_pos > isize) {
  314. spin_unlock(&ip->i_flags_lock);
  315. if (!drained_dio) {
  316. if (*iolock == XFS_IOLOCK_SHARED) {
  317. xfs_iunlock(ip, *iolock);
  318. *iolock = XFS_IOLOCK_EXCL;
  319. xfs_ilock(ip, *iolock);
  320. iov_iter_reexpand(from, count);
  321. }
  322. /*
  323. * We now have an IO submission barrier in place, but
  324. * AIO can do EOF updates during IO completion and hence
  325. * we now need to wait for all of them to drain. Non-AIO
  326. * DIO will have drained before we are given the
  327. * XFS_IOLOCK_EXCL, and so for most cases this wait is a
  328. * no-op.
  329. */
  330. inode_dio_wait(inode);
  331. drained_dio = true;
  332. goto restart;
  333. }
  334. trace_xfs_zero_eof(ip, isize, iocb->ki_pos - isize);
  335. error = iomap_zero_range(inode, isize, iocb->ki_pos - isize,
  336. NULL, &xfs_buffered_write_iomap_ops);
  337. if (error)
  338. return error;
  339. } else
  340. spin_unlock(&ip->i_flags_lock);
  341. /*
  342. * Updating the timestamps will grab the ilock again from
  343. * xfs_fs_dirty_inode, so we have to call it after dropping the
  344. * lock above. Eventually we should look into a way to avoid
  345. * the pointless lock roundtrip.
  346. */
  347. return file_modified(file);
  348. }
  349. static int
  350. xfs_dio_write_end_io(
  351. struct kiocb *iocb,
  352. ssize_t size,
  353. int error,
  354. unsigned flags)
  355. {
  356. struct inode *inode = file_inode(iocb->ki_filp);
  357. struct xfs_inode *ip = XFS_I(inode);
  358. loff_t offset = iocb->ki_pos;
  359. unsigned int nofs_flag;
  360. trace_xfs_end_io_direct_write(ip, offset, size);
  361. if (XFS_FORCED_SHUTDOWN(ip->i_mount))
  362. return -EIO;
  363. if (error)
  364. return error;
  365. if (!size)
  366. return 0;
  367. /*
  368. * Capture amount written on completion as we can't reliably account
  369. * for it on submission.
  370. */
  371. XFS_STATS_ADD(ip->i_mount, xs_write_bytes, size);
  372. /*
  373. * We can allocate memory here while doing writeback on behalf of
  374. * memory reclaim. To avoid memory allocation deadlocks set the
  375. * task-wide nofs context for the following operations.
  376. */
  377. nofs_flag = memalloc_nofs_save();
  378. if (flags & IOMAP_DIO_COW) {
  379. error = xfs_reflink_end_cow(ip, offset, size);
  380. if (error)
  381. goto out;
  382. }
  383. /*
  384. * Unwritten conversion updates the in-core isize after extent
  385. * conversion but before updating the on-disk size. Updating isize any
  386. * earlier allows a racing dio read to find unwritten extents before
  387. * they are converted.
  388. */
  389. if (flags & IOMAP_DIO_UNWRITTEN) {
  390. error = xfs_iomap_write_unwritten(ip, offset, size, true);
  391. goto out;
  392. }
  393. /*
  394. * We need to update the in-core inode size here so that we don't end up
  395. * with the on-disk inode size being outside the in-core inode size. We
  396. * have no other method of updating EOF for AIO, so always do it here
  397. * if necessary.
  398. *
  399. * We need to lock the test/set EOF update as we can be racing with
  400. * other IO completions here to update the EOF. Failing to serialise
  401. * here can result in EOF moving backwards and Bad Things Happen when
  402. * that occurs.
  403. */
  404. spin_lock(&ip->i_flags_lock);
  405. if (offset + size > i_size_read(inode)) {
  406. i_size_write(inode, offset + size);
  407. spin_unlock(&ip->i_flags_lock);
  408. error = xfs_setfilesize(ip, offset, size);
  409. } else {
  410. spin_unlock(&ip->i_flags_lock);
  411. }
  412. out:
  413. memalloc_nofs_restore(nofs_flag);
  414. return error;
  415. }
  416. static const struct iomap_dio_ops xfs_dio_write_ops = {
  417. .end_io = xfs_dio_write_end_io,
  418. };
  419. /*
  420. * xfs_file_dio_aio_write - handle direct IO writes
  421. *
  422. * Lock the inode appropriately to prepare for and issue a direct IO write.
  423. * By separating it from the buffered write path we remove all the tricky to
  424. * follow locking changes and looping.
  425. *
  426. * If there are cached pages or we're extending the file, we need IOLOCK_EXCL
  427. * until we're sure the bytes at the new EOF have been zeroed and/or the cached
  428. * pages are flushed out.
  429. *
  430. * In most cases the direct IO writes will be done holding IOLOCK_SHARED
  431. * allowing them to be done in parallel with reads and other direct IO writes.
  432. * However, if the IO is not aligned to filesystem blocks, the direct IO layer
  433. * needs to do sub-block zeroing and that requires serialisation against other
  434. * direct IOs to the same block. In this case we need to serialise the
  435. * submission of the unaligned IOs so that we don't get racing block zeroing in
  436. * the dio layer. To avoid the problem with aio, we also need to wait for
  437. * outstanding IOs to complete so that unwritten extent conversion is completed
  438. * before we try to map the overlapping block. This is currently implemented by
  439. * hitting it with a big hammer (i.e. inode_dio_wait()).
  440. *
  441. * Returns with locks held indicated by @iolock and errors indicated by
  442. * negative return values.
  443. */
  444. STATIC ssize_t
  445. xfs_file_dio_aio_write(
  446. struct kiocb *iocb,
  447. struct iov_iter *from)
  448. {
  449. struct file *file = iocb->ki_filp;
  450. struct address_space *mapping = file->f_mapping;
  451. struct inode *inode = mapping->host;
  452. struct xfs_inode *ip = XFS_I(inode);
  453. struct xfs_mount *mp = ip->i_mount;
  454. ssize_t ret = 0;
  455. int unaligned_io = 0;
  456. int iolock;
  457. size_t count = iov_iter_count(from);
  458. struct xfs_buftarg *target = xfs_inode_buftarg(ip);
  459. /* DIO must be aligned to device logical sector size */
  460. if ((iocb->ki_pos | count) & target->bt_logical_sectormask)
  461. return -EINVAL;
  462. /*
  463. * Don't take the exclusive iolock here unless the I/O is unaligned to
  464. * the file system block size. We don't need to consider the EOF
  465. * extension case here because xfs_file_aio_write_checks() will relock
  466. * the inode as necessary for EOF zeroing cases and fill out the new
  467. * inode size as appropriate.
  468. */
  469. if ((iocb->ki_pos & mp->m_blockmask) ||
  470. ((iocb->ki_pos + count) & mp->m_blockmask)) {
  471. unaligned_io = 1;
  472. /*
  473. * We can't properly handle unaligned direct I/O to reflink
  474. * files yet, as we can't unshare a partial block.
  475. */
  476. if (xfs_is_cow_inode(ip)) {
  477. trace_xfs_reflink_bounce_dio_write(ip, iocb->ki_pos, count);
  478. return -ENOTBLK;
  479. }
  480. iolock = XFS_IOLOCK_EXCL;
  481. } else {
  482. iolock = XFS_IOLOCK_SHARED;
  483. }
  484. if (iocb->ki_flags & IOCB_NOWAIT) {
  485. /* unaligned dio always waits, bail */
  486. if (unaligned_io)
  487. return -EAGAIN;
  488. if (!xfs_ilock_nowait(ip, iolock))
  489. return -EAGAIN;
  490. } else {
  491. xfs_ilock(ip, iolock);
  492. }
  493. ret = xfs_file_aio_write_checks(iocb, from, &iolock);
  494. if (ret)
  495. goto out;
  496. count = iov_iter_count(from);
  497. /*
  498. * If we are doing unaligned IO, we can't allow any other overlapping IO
  499. * in-flight at the same time or we risk data corruption. Wait for all
  500. * other IO to drain before we submit. If the IO is aligned, demote the
  501. * iolock if we had to take the exclusive lock in
  502. * xfs_file_aio_write_checks() for other reasons.
  503. */
  504. if (unaligned_io) {
  505. inode_dio_wait(inode);
  506. } else if (iolock == XFS_IOLOCK_EXCL) {
  507. xfs_ilock_demote(ip, XFS_IOLOCK_EXCL);
  508. iolock = XFS_IOLOCK_SHARED;
  509. }
  510. trace_xfs_file_direct_write(ip, count, iocb->ki_pos);
  511. /*
  512. * If unaligned, this is the only IO in-flight. Wait on it before we
  513. * release the iolock to prevent subsequent overlapping IO.
  514. */
  515. ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
  516. &xfs_dio_write_ops,
  517. is_sync_kiocb(iocb) || unaligned_io);
  518. out:
  519. xfs_iunlock(ip, iolock);
  520. /*
  521. * No fallback to buffered IO after short writes for XFS, direct I/O
  522. * will either complete fully or return an error.
  523. */
  524. ASSERT(ret < 0 || ret == count);
  525. return ret;
  526. }
  527. static noinline ssize_t
  528. xfs_file_dax_write(
  529. struct kiocb *iocb,
  530. struct iov_iter *from)
  531. {
  532. struct inode *inode = iocb->ki_filp->f_mapping->host;
  533. struct xfs_inode *ip = XFS_I(inode);
  534. int iolock = XFS_IOLOCK_EXCL;
  535. ssize_t ret, error = 0;
  536. size_t count;
  537. loff_t pos;
  538. if (iocb->ki_flags & IOCB_NOWAIT) {
  539. if (!xfs_ilock_nowait(ip, iolock))
  540. return -EAGAIN;
  541. } else {
  542. xfs_ilock(ip, iolock);
  543. }
  544. ret = xfs_file_aio_write_checks(iocb, from, &iolock);
  545. if (ret)
  546. goto out;
  547. pos = iocb->ki_pos;
  548. count = iov_iter_count(from);
  549. trace_xfs_file_dax_write(ip, count, pos);
  550. ret = dax_iomap_rw(iocb, from, &xfs_direct_write_iomap_ops);
  551. if (ret > 0 && iocb->ki_pos > i_size_read(inode)) {
  552. i_size_write(inode, iocb->ki_pos);
  553. error = xfs_setfilesize(ip, pos, ret);
  554. }
  555. out:
  556. xfs_iunlock(ip, iolock);
  557. if (error)
  558. return error;
  559. if (ret > 0) {
  560. XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
  561. /* Handle various SYNC-type writes */
  562. ret = generic_write_sync(iocb, ret);
  563. }
  564. return ret;
  565. }
  566. STATIC ssize_t
  567. xfs_file_buffered_aio_write(
  568. struct kiocb *iocb,
  569. struct iov_iter *from)
  570. {
  571. struct file *file = iocb->ki_filp;
  572. struct address_space *mapping = file->f_mapping;
  573. struct inode *inode = mapping->host;
  574. struct xfs_inode *ip = XFS_I(inode);
  575. ssize_t ret;
  576. int enospc = 0;
  577. int iolock;
  578. if (iocb->ki_flags & IOCB_NOWAIT)
  579. return -EOPNOTSUPP;
  580. write_retry:
  581. iolock = XFS_IOLOCK_EXCL;
  582. xfs_ilock(ip, iolock);
  583. ret = xfs_file_aio_write_checks(iocb, from, &iolock);
  584. if (ret)
  585. goto out;
  586. /* We can write back this queue in page reclaim */
  587. current->backing_dev_info = inode_to_bdi(inode);
  588. trace_xfs_file_buffered_write(ip, iov_iter_count(from), iocb->ki_pos);
  589. ret = iomap_file_buffered_write(iocb, from,
  590. &xfs_buffered_write_iomap_ops);
  591. if (likely(ret >= 0))
  592. iocb->ki_pos += ret;
  593. /*
  594. * If we hit a space limit, try to free up some lingering preallocated
  595. * space before returning an error. In the case of ENOSPC, first try to
  596. * write back all dirty inodes to free up some of the excess reserved
  597. * metadata space. This reduces the chances that the eofblocks scan
  598. * waits on dirty mappings. Since xfs_flush_inodes() is serialized, this
  599. * also behaves as a filter to prevent too many eofblocks scans from
  600. * running at the same time.
  601. */
  602. if (ret == -EDQUOT && !enospc) {
  603. xfs_iunlock(ip, iolock);
  604. enospc = xfs_inode_free_quota_eofblocks(ip);
  605. if (enospc)
  606. goto write_retry;
  607. enospc = xfs_inode_free_quota_cowblocks(ip);
  608. if (enospc)
  609. goto write_retry;
  610. iolock = 0;
  611. } else if (ret == -ENOSPC && !enospc) {
  612. struct xfs_eofblocks eofb = {0};
  613. enospc = 1;
  614. xfs_flush_inodes(ip->i_mount);
  615. xfs_iunlock(ip, iolock);
  616. eofb.eof_flags = XFS_EOF_FLAGS_SYNC;
  617. xfs_icache_free_eofblocks(ip->i_mount, &eofb);
  618. xfs_icache_free_cowblocks(ip->i_mount, &eofb);
  619. goto write_retry;
  620. }
  621. current->backing_dev_info = NULL;
  622. out:
  623. if (iolock)
  624. xfs_iunlock(ip, iolock);
  625. if (ret > 0) {
  626. XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
  627. /* Handle various SYNC-type writes */
  628. ret = generic_write_sync(iocb, ret);
  629. }
  630. return ret;
  631. }
  632. STATIC ssize_t
  633. xfs_file_write_iter(
  634. struct kiocb *iocb,
  635. struct iov_iter *from)
  636. {
  637. struct file *file = iocb->ki_filp;
  638. struct address_space *mapping = file->f_mapping;
  639. struct inode *inode = mapping->host;
  640. struct xfs_inode *ip = XFS_I(inode);
  641. ssize_t ret;
  642. size_t ocount = iov_iter_count(from);
  643. XFS_STATS_INC(ip->i_mount, xs_write_calls);
  644. if (ocount == 0)
  645. return 0;
  646. if (XFS_FORCED_SHUTDOWN(ip->i_mount))
  647. return -EIO;
  648. if (IS_DAX(inode))
  649. return xfs_file_dax_write(iocb, from);
  650. if (iocb->ki_flags & IOCB_DIRECT) {
  651. /*
  652. * Allow a directio write to fall back to a buffered
  653. * write *only* in the case that we're doing a reflink
  654. * CoW. In all other directio scenarios we do not
  655. * allow an operation to fall back to buffered mode.
  656. */
  657. ret = xfs_file_dio_aio_write(iocb, from);
  658. if (ret != -ENOTBLK)
  659. return ret;
  660. }
  661. return xfs_file_buffered_aio_write(iocb, from);
  662. }
  663. static void
  664. xfs_wait_dax_page(
  665. struct inode *inode)
  666. {
  667. struct xfs_inode *ip = XFS_I(inode);
  668. xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
  669. schedule();
  670. xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
  671. }
  672. static int
  673. xfs_break_dax_layouts(
  674. struct inode *inode,
  675. bool *retry)
  676. {
  677. struct page *page;
  678. ASSERT(xfs_isilocked(XFS_I(inode), XFS_MMAPLOCK_EXCL));
  679. page = dax_layout_busy_page(inode->i_mapping);
  680. if (!page)
  681. return 0;
  682. *retry = true;
  683. return ___wait_var_event(&page->_refcount,
  684. atomic_read(&page->_refcount) == 1, TASK_INTERRUPTIBLE,
  685. 0, 0, xfs_wait_dax_page(inode));
  686. }
  687. int
  688. xfs_break_layouts(
  689. struct inode *inode,
  690. uint *iolock,
  691. enum layout_break_reason reason)
  692. {
  693. bool retry;
  694. int error;
  695. ASSERT(xfs_isilocked(XFS_I(inode), XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL));
  696. do {
  697. retry = false;
  698. switch (reason) {
  699. case BREAK_UNMAP:
  700. error = xfs_break_dax_layouts(inode, &retry);
  701. if (error || retry)
  702. break;
  703. /* fall through */
  704. case BREAK_WRITE:
  705. error = xfs_break_leased_layouts(inode, iolock, &retry);
  706. break;
  707. default:
  708. WARN_ON_ONCE(1);
  709. error = -EINVAL;
  710. }
  711. } while (error == 0 && retry);
  712. return error;
  713. }
  714. #define XFS_FALLOC_FL_SUPPORTED \
  715. (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \
  716. FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | \
  717. FALLOC_FL_INSERT_RANGE | FALLOC_FL_UNSHARE_RANGE)
  718. STATIC long
  719. xfs_file_fallocate(
  720. struct file *file,
  721. int mode,
  722. loff_t offset,
  723. loff_t len)
  724. {
  725. struct inode *inode = file_inode(file);
  726. struct xfs_inode *ip = XFS_I(inode);
  727. long error;
  728. enum xfs_prealloc_flags flags = 0;
  729. uint iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL;
  730. loff_t new_size = 0;
  731. bool do_file_insert = false;
  732. if (!S_ISREG(inode->i_mode))
  733. return -EINVAL;
  734. if (mode & ~XFS_FALLOC_FL_SUPPORTED)
  735. return -EOPNOTSUPP;
  736. xfs_ilock(ip, iolock);
  737. error = xfs_break_layouts(inode, &iolock, BREAK_UNMAP);
  738. if (error)
  739. goto out_unlock;
  740. /*
  741. * Must wait for all AIO to complete before we continue as AIO can
  742. * change the file size on completion without holding any locks we
  743. * currently hold. We must do this first because AIO can update both
  744. * the on disk and in memory inode sizes, and the operations that follow
  745. * require the in-memory size to be fully up-to-date.
  746. */
  747. inode_dio_wait(inode);
  748. /*
  749. * Now AIO and DIO has drained we flush and (if necessary) invalidate
  750. * the cached range over the first operation we are about to run.
  751. *
  752. * We care about zero and collapse here because they both run a hole
  753. * punch over the range first. Because that can zero data, and the range
  754. * of invalidation for the shift operations is much larger, we still do
  755. * the required flush for collapse in xfs_prepare_shift().
  756. *
  757. * Insert has the same range requirements as collapse, and we extend the
  758. * file first which can zero data. Hence insert has the same
  759. * flush/invalidate requirements as collapse and so they are both
  760. * handled at the right time by xfs_prepare_shift().
  761. */
  762. if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE |
  763. FALLOC_FL_COLLAPSE_RANGE)) {
  764. error = xfs_flush_unmap_range(ip, offset, len);
  765. if (error)
  766. goto out_unlock;
  767. }
  768. if (mode & FALLOC_FL_PUNCH_HOLE) {
  769. error = xfs_free_file_space(ip, offset, len);
  770. if (error)
  771. goto out_unlock;
  772. } else if (mode & FALLOC_FL_COLLAPSE_RANGE) {
  773. if (!xfs_is_falloc_aligned(ip, offset, len)) {
  774. error = -EINVAL;
  775. goto out_unlock;
  776. }
  777. /*
  778. * There is no need to overlap collapse range with EOF,
  779. * in which case it is effectively a truncate operation
  780. */
  781. if (offset + len >= i_size_read(inode)) {
  782. error = -EINVAL;
  783. goto out_unlock;
  784. }
  785. new_size = i_size_read(inode) - len;
  786. error = xfs_collapse_file_space(ip, offset, len);
  787. if (error)
  788. goto out_unlock;
  789. } else if (mode & FALLOC_FL_INSERT_RANGE) {
  790. loff_t isize = i_size_read(inode);
  791. if (!xfs_is_falloc_aligned(ip, offset, len)) {
  792. error = -EINVAL;
  793. goto out_unlock;
  794. }
  795. /*
  796. * New inode size must not exceed ->s_maxbytes, accounting for
  797. * possible signed overflow.
  798. */
  799. if (inode->i_sb->s_maxbytes - isize < len) {
  800. error = -EFBIG;
  801. goto out_unlock;
  802. }
  803. new_size = isize + len;
  804. /* Offset should be less than i_size */
  805. if (offset >= isize) {
  806. error = -EINVAL;
  807. goto out_unlock;
  808. }
  809. do_file_insert = true;
  810. } else {
  811. flags |= XFS_PREALLOC_SET;
  812. if (!(mode & FALLOC_FL_KEEP_SIZE) &&
  813. offset + len > i_size_read(inode)) {
  814. new_size = offset + len;
  815. error = inode_newsize_ok(inode, new_size);
  816. if (error)
  817. goto out_unlock;
  818. }
  819. if (mode & FALLOC_FL_ZERO_RANGE) {
  820. /*
  821. * Punch a hole and prealloc the range. We use a hole
  822. * punch rather than unwritten extent conversion for two
  823. * reasons:
  824. *
  825. * 1.) Hole punch handles partial block zeroing for us.
  826. * 2.) If prealloc returns ENOSPC, the file range is
  827. * still zero-valued by virtue of the hole punch.
  828. */
  829. unsigned int blksize = i_blocksize(inode);
  830. trace_xfs_zero_file_space(ip);
  831. error = xfs_free_file_space(ip, offset, len);
  832. if (error)
  833. goto out_unlock;
  834. len = round_up(offset + len, blksize) -
  835. round_down(offset, blksize);
  836. offset = round_down(offset, blksize);
  837. } else if (mode & FALLOC_FL_UNSHARE_RANGE) {
  838. error = xfs_reflink_unshare(ip, offset, len);
  839. if (error)
  840. goto out_unlock;
  841. } else {
  842. /*
  843. * If always_cow mode we can't use preallocations and
  844. * thus should not create them.
  845. */
  846. if (xfs_is_always_cow_inode(ip)) {
  847. error = -EOPNOTSUPP;
  848. goto out_unlock;
  849. }
  850. }
  851. if (!xfs_is_always_cow_inode(ip)) {
  852. error = xfs_alloc_file_space(ip, offset, len,
  853. XFS_BMAPI_PREALLOC);
  854. if (error)
  855. goto out_unlock;
  856. }
  857. }
  858. if (file->f_flags & O_DSYNC)
  859. flags |= XFS_PREALLOC_SYNC;
  860. error = xfs_update_prealloc_flags(ip, flags);
  861. if (error)
  862. goto out_unlock;
  863. /* Change file size if needed */
  864. if (new_size) {
  865. struct iattr iattr;
  866. iattr.ia_valid = ATTR_SIZE;
  867. iattr.ia_size = new_size;
  868. error = xfs_vn_setattr_size(file_dentry(file), &iattr);
  869. if (error)
  870. goto out_unlock;
  871. }
  872. /*
  873. * Perform hole insertion now that the file size has been
  874. * updated so that if we crash during the operation we don't
  875. * leave shifted extents past EOF and hence losing access to
  876. * the data that is contained within them.
  877. */
  878. if (do_file_insert)
  879. error = xfs_insert_file_space(ip, offset, len);
  880. out_unlock:
  881. xfs_iunlock(ip, iolock);
  882. return error;
  883. }
  884. STATIC int
  885. xfs_file_fadvise(
  886. struct file *file,
  887. loff_t start,
  888. loff_t end,
  889. int advice)
  890. {
  891. struct xfs_inode *ip = XFS_I(file_inode(file));
  892. int ret;
  893. int lockflags = 0;
  894. /*
  895. * Operations creating pages in page cache need protection from hole
  896. * punching and similar ops
  897. */
  898. if (advice == POSIX_FADV_WILLNEED) {
  899. lockflags = XFS_IOLOCK_SHARED;
  900. xfs_ilock(ip, lockflags);
  901. }
  902. ret = generic_fadvise(file, start, end, advice);
  903. if (lockflags)
  904. xfs_iunlock(ip, lockflags);
  905. return ret;
  906. }
  907. /* Does this file, inode, or mount want synchronous writes? */
  908. static inline bool xfs_file_sync_writes(struct file *filp)
  909. {
  910. struct xfs_inode *ip = XFS_I(file_inode(filp));
  911. if (ip->i_mount->m_flags & XFS_MOUNT_WSYNC)
  912. return true;
  913. if (filp->f_flags & (__O_SYNC | O_DSYNC))
  914. return true;
  915. if (IS_SYNC(file_inode(filp)))
  916. return true;
  917. return false;
  918. }
  919. STATIC loff_t
  920. xfs_file_remap_range(
  921. struct file *file_in,
  922. loff_t pos_in,
  923. struct file *file_out,
  924. loff_t pos_out,
  925. loff_t len,
  926. unsigned int remap_flags)
  927. {
  928. struct inode *inode_in = file_inode(file_in);
  929. struct xfs_inode *src = XFS_I(inode_in);
  930. struct inode *inode_out = file_inode(file_out);
  931. struct xfs_inode *dest = XFS_I(inode_out);
  932. struct xfs_mount *mp = src->i_mount;
  933. loff_t remapped = 0;
  934. xfs_extlen_t cowextsize;
  935. int ret;
  936. if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY))
  937. return -EINVAL;
  938. if (!xfs_sb_version_hasreflink(&mp->m_sb))
  939. return -EOPNOTSUPP;
  940. if (XFS_FORCED_SHUTDOWN(mp))
  941. return -EIO;
  942. /* Prepare and then clone file data. */
  943. ret = xfs_reflink_remap_prep(file_in, pos_in, file_out, pos_out,
  944. &len, remap_flags);
  945. if (ret || len == 0)
  946. return ret;
  947. trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out);
  948. ret = xfs_reflink_remap_blocks(src, pos_in, dest, pos_out, len,
  949. &remapped);
  950. if (ret)
  951. goto out_unlock;
  952. /*
  953. * Carry the cowextsize hint from src to dest if we're sharing the
  954. * entire source file to the entire destination file, the source file
  955. * has a cowextsize hint, and the destination file does not.
  956. */
  957. cowextsize = 0;
  958. if (pos_in == 0 && len == i_size_read(inode_in) &&
  959. (src->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE) &&
  960. pos_out == 0 && len >= i_size_read(inode_out) &&
  961. !(dest->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE))
  962. cowextsize = src->i_d.di_cowextsize;
  963. ret = xfs_reflink_update_dest(dest, pos_out + len, cowextsize,
  964. remap_flags);
  965. if (ret)
  966. goto out_unlock;
  967. if (xfs_file_sync_writes(file_in) || xfs_file_sync_writes(file_out))
  968. xfs_log_force_inode(dest);
  969. out_unlock:
  970. xfs_iunlock2_io_mmap(src, dest);
  971. if (ret)
  972. trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_);
  973. return remapped > 0 ? remapped : ret;
  974. }
  975. STATIC int
  976. xfs_file_open(
  977. struct inode *inode,
  978. struct file *file)
  979. {
  980. if (!(file->f_flags & O_LARGEFILE) && i_size_read(inode) > MAX_NON_LFS)
  981. return -EFBIG;
  982. if (XFS_FORCED_SHUTDOWN(XFS_M(inode->i_sb)))
  983. return -EIO;
  984. file->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC;
  985. return 0;
  986. }
  987. STATIC int
  988. xfs_dir_open(
  989. struct inode *inode,
  990. struct file *file)
  991. {
  992. struct xfs_inode *ip = XFS_I(inode);
  993. int mode;
  994. int error;
  995. error = xfs_file_open(inode, file);
  996. if (error)
  997. return error;
  998. /*
  999. * If there are any blocks, read-ahead block 0 as we're almost
  1000. * certain to have the next operation be a read there.
  1001. */
  1002. mode = xfs_ilock_data_map_shared(ip);
  1003. if (ip->i_df.if_nextents > 0)
  1004. error = xfs_dir3_data_readahead(ip, 0, 0);
  1005. xfs_iunlock(ip, mode);
  1006. return error;
  1007. }
  1008. STATIC int
  1009. xfs_file_release(
  1010. struct inode *inode,
  1011. struct file *filp)
  1012. {
  1013. return xfs_release(XFS_I(inode));
  1014. }
  1015. STATIC int
  1016. xfs_file_readdir(
  1017. struct file *file,
  1018. struct dir_context *ctx)
  1019. {
  1020. struct inode *inode = file_inode(file);
  1021. xfs_inode_t *ip = XFS_I(inode);
  1022. size_t bufsize;
  1023. /*
  1024. * The Linux API doesn't pass down the total size of the buffer
  1025. * we read into down to the filesystem. With the filldir concept
  1026. * it's not needed for correct information, but the XFS dir2 leaf
  1027. * code wants an estimate of the buffer size to calculate it's
  1028. * readahead window and size the buffers used for mapping to
  1029. * physical blocks.
  1030. *
  1031. * Try to give it an estimate that's good enough, maybe at some
  1032. * point we can change the ->readdir prototype to include the
  1033. * buffer size. For now we use the current glibc buffer size.
  1034. */
  1035. bufsize = (size_t)min_t(loff_t, XFS_READDIR_BUFSIZE, ip->i_d.di_size);
  1036. return xfs_readdir(NULL, ip, ctx, bufsize);
  1037. }
  1038. STATIC loff_t
  1039. xfs_file_llseek(
  1040. struct file *file,
  1041. loff_t offset,
  1042. int whence)
  1043. {
  1044. struct inode *inode = file->f_mapping->host;
  1045. if (XFS_FORCED_SHUTDOWN(XFS_I(inode)->i_mount))
  1046. return -EIO;
  1047. switch (whence) {
  1048. default:
  1049. return generic_file_llseek(file, offset, whence);
  1050. case SEEK_HOLE:
  1051. offset = iomap_seek_hole(inode, offset, &xfs_seek_iomap_ops);
  1052. break;
  1053. case SEEK_DATA:
  1054. offset = iomap_seek_data(inode, offset, &xfs_seek_iomap_ops);
  1055. break;
  1056. }
  1057. if (offset < 0)
  1058. return offset;
  1059. return vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
  1060. }
  1061. /*
  1062. * Locking for serialisation of IO during page faults. This results in a lock
  1063. * ordering of:
  1064. *
  1065. * mmap_lock (MM)
  1066. * sb_start_pagefault(vfs, freeze)
  1067. * i_mmaplock (XFS - truncate serialisation)
  1068. * page_lock (MM)
  1069. * i_lock (XFS - extent map serialisation)
  1070. */
  1071. static vm_fault_t
  1072. __xfs_filemap_fault(
  1073. struct vm_fault *vmf,
  1074. enum page_entry_size pe_size,
  1075. bool write_fault)
  1076. {
  1077. struct inode *inode = file_inode(vmf->vma->vm_file);
  1078. struct xfs_inode *ip = XFS_I(inode);
  1079. vm_fault_t ret;
  1080. trace_xfs_filemap_fault(ip, pe_size, write_fault);
  1081. if (write_fault) {
  1082. sb_start_pagefault(inode->i_sb);
  1083. file_update_time(vmf->vma->vm_file);
  1084. }
  1085. xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
  1086. if (IS_DAX(inode)) {
  1087. pfn_t pfn;
  1088. ret = dax_iomap_fault(vmf, pe_size, &pfn, NULL,
  1089. (write_fault && !vmf->cow_page) ?
  1090. &xfs_direct_write_iomap_ops :
  1091. &xfs_read_iomap_ops);
  1092. if (ret & VM_FAULT_NEEDDSYNC)
  1093. ret = dax_finish_sync_fault(vmf, pe_size, pfn);
  1094. } else {
  1095. if (write_fault)
  1096. ret = iomap_page_mkwrite(vmf,
  1097. &xfs_buffered_write_iomap_ops);
  1098. else
  1099. ret = filemap_fault(vmf);
  1100. }
  1101. xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
  1102. if (write_fault)
  1103. sb_end_pagefault(inode->i_sb);
  1104. return ret;
  1105. }
  1106. static inline bool
  1107. xfs_is_write_fault(
  1108. struct vm_fault *vmf)
  1109. {
  1110. return (vmf->flags & FAULT_FLAG_WRITE) &&
  1111. (vmf->vma->vm_flags & VM_SHARED);
  1112. }
  1113. static vm_fault_t
  1114. xfs_filemap_fault(
  1115. struct vm_fault *vmf)
  1116. {
  1117. /* DAX can shortcut the normal fault path on write faults! */
  1118. return __xfs_filemap_fault(vmf, PE_SIZE_PTE,
  1119. IS_DAX(file_inode(vmf->vma->vm_file)) &&
  1120. xfs_is_write_fault(vmf));
  1121. }
  1122. static vm_fault_t
  1123. xfs_filemap_huge_fault(
  1124. struct vm_fault *vmf,
  1125. enum page_entry_size pe_size)
  1126. {
  1127. if (!IS_DAX(file_inode(vmf->vma->vm_file)))
  1128. return VM_FAULT_FALLBACK;
  1129. /* DAX can shortcut the normal fault path on write faults! */
  1130. return __xfs_filemap_fault(vmf, pe_size,
  1131. xfs_is_write_fault(vmf));
  1132. }
  1133. static vm_fault_t
  1134. xfs_filemap_page_mkwrite(
  1135. struct vm_fault *vmf)
  1136. {
  1137. return __xfs_filemap_fault(vmf, PE_SIZE_PTE, true);
  1138. }
  1139. /*
  1140. * pfn_mkwrite was originally intended to ensure we capture time stamp updates
  1141. * on write faults. In reality, it needs to serialise against truncate and
  1142. * prepare memory for writing so handle is as standard write fault.
  1143. */
  1144. static vm_fault_t
  1145. xfs_filemap_pfn_mkwrite(
  1146. struct vm_fault *vmf)
  1147. {
  1148. return __xfs_filemap_fault(vmf, PE_SIZE_PTE, true);
  1149. }
  1150. static vm_fault_t
  1151. xfs_filemap_map_pages(
  1152. struct vm_fault *vmf,
  1153. pgoff_t start_pgoff,
  1154. pgoff_t end_pgoff)
  1155. {
  1156. struct inode *inode = file_inode(vmf->vma->vm_file);
  1157. vm_fault_t ret;
  1158. xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
  1159. ret = filemap_map_pages(vmf, start_pgoff, end_pgoff);
  1160. xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
  1161. return ret;
  1162. }
  1163. static const struct vm_operations_struct xfs_file_vm_ops = {
  1164. .fault = xfs_filemap_fault,
  1165. .huge_fault = xfs_filemap_huge_fault,
  1166. .map_pages = xfs_filemap_map_pages,
  1167. .page_mkwrite = xfs_filemap_page_mkwrite,
  1168. .pfn_mkwrite = xfs_filemap_pfn_mkwrite,
  1169. };
  1170. STATIC int
  1171. xfs_file_mmap(
  1172. struct file *file,
  1173. struct vm_area_struct *vma)
  1174. {
  1175. struct inode *inode = file_inode(file);
  1176. struct xfs_buftarg *target = xfs_inode_buftarg(XFS_I(inode));
  1177. /*
  1178. * We don't support synchronous mappings for non-DAX files and
  1179. * for DAX files if underneath dax_device is not synchronous.
  1180. */
  1181. if (!daxdev_mapping_supported(vma, target->bt_daxdev))
  1182. return -EOPNOTSUPP;
  1183. file_accessed(file);
  1184. vma->vm_ops = &xfs_file_vm_ops;
  1185. if (IS_DAX(inode))
  1186. vma->vm_flags |= VM_HUGEPAGE;
  1187. return 0;
  1188. }
  1189. const struct file_operations xfs_file_operations = {
  1190. .llseek = xfs_file_llseek,
  1191. .read_iter = xfs_file_read_iter,
  1192. .write_iter = xfs_file_write_iter,
  1193. .splice_read = generic_file_splice_read,
  1194. .splice_write = iter_file_splice_write,
  1195. .iopoll = iomap_dio_iopoll,
  1196. .unlocked_ioctl = xfs_file_ioctl,
  1197. #ifdef CONFIG_COMPAT
  1198. .compat_ioctl = xfs_file_compat_ioctl,
  1199. #endif
  1200. .mmap = xfs_file_mmap,
  1201. .mmap_supported_flags = MAP_SYNC,
  1202. .open = xfs_file_open,
  1203. .release = xfs_file_release,
  1204. .fsync = xfs_file_fsync,
  1205. .get_unmapped_area = thp_get_unmapped_area,
  1206. .fallocate = xfs_file_fallocate,
  1207. .fadvise = xfs_file_fadvise,
  1208. .remap_file_range = xfs_file_remap_range,
  1209. };
  1210. const struct file_operations xfs_dir_file_operations = {
  1211. .open = xfs_dir_open,
  1212. .read = generic_read_dir,
  1213. .iterate_shared = xfs_file_readdir,
  1214. .llseek = generic_file_llseek,
  1215. .unlocked_ioctl = xfs_file_ioctl,
  1216. #ifdef CONFIG_COMPAT
  1217. .compat_ioctl = xfs_file_compat_ioctl,
  1218. #endif
  1219. .fsync = xfs_dir_fsync,
  1220. };