buffered-io.c 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2010 Red Hat, Inc.
  4. * Copyright (C) 2016-2019 Christoph Hellwig.
  5. */
  6. #include <linux/module.h>
  7. #include <linux/compiler.h>
  8. #include <linux/fs.h>
  9. #include <linux/iomap.h>
  10. #include <linux/pagemap.h>
  11. #include <linux/uio.h>
  12. #include <linux/buffer_head.h>
  13. #include <linux/dax.h>
  14. #include <linux/writeback.h>
  15. #include <linux/list_sort.h>
  16. #include <linux/swap.h>
  17. #include <linux/bio.h>
  18. #include <linux/sched/signal.h>
  19. #include <linux/migrate.h>
  20. #include "trace.h"
  21. #include "../internal.h"
  22. /*
  23. * Structure allocated for each page or THP when block size < page size
  24. * to track sub-page uptodate status and I/O completions.
  25. */
  26. struct iomap_page {
  27. atomic_t read_bytes_pending;
  28. atomic_t write_bytes_pending;
  29. spinlock_t uptodate_lock;
  30. unsigned long uptodate[];
  31. };
  32. static inline struct iomap_page *to_iomap_page(struct page *page)
  33. {
  34. /*
  35. * per-block data is stored in the head page. Callers should
  36. * not be dealing with tail pages (and if they are, they can
  37. * call thp_head() first.
  38. */
  39. VM_BUG_ON_PGFLAGS(PageTail(page), page);
  40. if (page_has_private(page))
  41. return (struct iomap_page *)page_private(page);
  42. return NULL;
  43. }
  44. static struct bio_set iomap_ioend_bioset;
  45. static struct iomap_page *
  46. iomap_page_create(struct inode *inode, struct page *page)
  47. {
  48. struct iomap_page *iop = to_iomap_page(page);
  49. unsigned int nr_blocks = i_blocks_per_page(inode, page);
  50. if (iop || nr_blocks <= 1)
  51. return iop;
  52. iop = kzalloc(struct_size(iop, uptodate, BITS_TO_LONGS(nr_blocks)),
  53. GFP_NOFS | __GFP_NOFAIL);
  54. spin_lock_init(&iop->uptodate_lock);
  55. if (PageUptodate(page))
  56. bitmap_fill(iop->uptodate, nr_blocks);
  57. attach_page_private(page, iop);
  58. return iop;
  59. }
  60. static void
  61. iomap_page_release(struct page *page)
  62. {
  63. struct iomap_page *iop = detach_page_private(page);
  64. unsigned int nr_blocks = i_blocks_per_page(page->mapping->host, page);
  65. if (!iop)
  66. return;
  67. WARN_ON_ONCE(atomic_read(&iop->read_bytes_pending));
  68. WARN_ON_ONCE(atomic_read(&iop->write_bytes_pending));
  69. WARN_ON_ONCE(bitmap_full(iop->uptodate, nr_blocks) !=
  70. PageUptodate(page));
  71. kfree(iop);
  72. }
  73. /*
  74. * Calculate the range inside the page that we actually need to read.
  75. */
  76. static void
  77. iomap_adjust_read_range(struct inode *inode, struct iomap_page *iop,
  78. loff_t *pos, loff_t length, unsigned *offp, unsigned *lenp)
  79. {
  80. loff_t orig_pos = *pos;
  81. loff_t isize = i_size_read(inode);
  82. unsigned block_bits = inode->i_blkbits;
  83. unsigned block_size = (1 << block_bits);
  84. unsigned poff = offset_in_page(*pos);
  85. unsigned plen = min_t(loff_t, PAGE_SIZE - poff, length);
  86. unsigned first = poff >> block_bits;
  87. unsigned last = (poff + plen - 1) >> block_bits;
  88. /*
  89. * If the block size is smaller than the page size we need to check the
  90. * per-block uptodate status and adjust the offset and length if needed
  91. * to avoid reading in already uptodate ranges.
  92. */
  93. if (iop) {
  94. unsigned int i;
  95. /* move forward for each leading block marked uptodate */
  96. for (i = first; i <= last; i++) {
  97. if (!test_bit(i, iop->uptodate))
  98. break;
  99. *pos += block_size;
  100. poff += block_size;
  101. plen -= block_size;
  102. first++;
  103. }
  104. /* truncate len if we find any trailing uptodate block(s) */
  105. for ( ; i <= last; i++) {
  106. if (test_bit(i, iop->uptodate)) {
  107. plen -= (last - i + 1) * block_size;
  108. last = i - 1;
  109. break;
  110. }
  111. }
  112. }
  113. /*
  114. * If the extent spans the block that contains the i_size we need to
  115. * handle both halves separately so that we properly zero data in the
  116. * page cache for blocks that are entirely outside of i_size.
  117. */
  118. if (orig_pos <= isize && orig_pos + length > isize) {
  119. unsigned end = offset_in_page(isize - 1) >> block_bits;
  120. if (first <= end && last > end)
  121. plen -= (last - end) * block_size;
  122. }
  123. *offp = poff;
  124. *lenp = plen;
  125. }
  126. static void
  127. iomap_iop_set_range_uptodate(struct page *page, unsigned off, unsigned len)
  128. {
  129. struct iomap_page *iop = to_iomap_page(page);
  130. struct inode *inode = page->mapping->host;
  131. unsigned first = off >> inode->i_blkbits;
  132. unsigned last = (off + len - 1) >> inode->i_blkbits;
  133. unsigned long flags;
  134. spin_lock_irqsave(&iop->uptodate_lock, flags);
  135. bitmap_set(iop->uptodate, first, last - first + 1);
  136. if (bitmap_full(iop->uptodate, i_blocks_per_page(inode, page)))
  137. SetPageUptodate(page);
  138. spin_unlock_irqrestore(&iop->uptodate_lock, flags);
  139. }
  140. static void
  141. iomap_set_range_uptodate(struct page *page, unsigned off, unsigned len)
  142. {
  143. if (PageError(page))
  144. return;
  145. if (page_has_private(page))
  146. iomap_iop_set_range_uptodate(page, off, len);
  147. else
  148. SetPageUptodate(page);
  149. }
  150. static void
  151. iomap_read_page_end_io(struct bio_vec *bvec, int error)
  152. {
  153. struct page *page = bvec->bv_page;
  154. struct iomap_page *iop = to_iomap_page(page);
  155. if (unlikely(error)) {
  156. ClearPageUptodate(page);
  157. SetPageError(page);
  158. } else {
  159. iomap_set_range_uptodate(page, bvec->bv_offset, bvec->bv_len);
  160. }
  161. if (!iop || atomic_sub_and_test(bvec->bv_len, &iop->read_bytes_pending))
  162. unlock_page(page);
  163. }
  164. static void
  165. iomap_read_end_io(struct bio *bio)
  166. {
  167. int error = blk_status_to_errno(bio->bi_status);
  168. struct bio_vec *bvec;
  169. struct bvec_iter_all iter_all;
  170. bio_for_each_segment_all(bvec, bio, iter_all)
  171. iomap_read_page_end_io(bvec, error);
  172. bio_put(bio);
  173. }
  174. struct iomap_readpage_ctx {
  175. struct page *cur_page;
  176. bool cur_page_in_bio;
  177. struct bio *bio;
  178. struct readahead_control *rac;
  179. };
  180. static void
  181. iomap_read_inline_data(struct inode *inode, struct page *page,
  182. struct iomap *iomap)
  183. {
  184. size_t size = i_size_read(inode);
  185. void *addr;
  186. if (PageUptodate(page))
  187. return;
  188. BUG_ON(page->index);
  189. BUG_ON(size > PAGE_SIZE - offset_in_page(iomap->inline_data));
  190. addr = kmap_atomic(page);
  191. memcpy(addr, iomap->inline_data, size);
  192. memset(addr + size, 0, PAGE_SIZE - size);
  193. kunmap_atomic(addr);
  194. SetPageUptodate(page);
  195. }
  196. static inline bool iomap_block_needs_zeroing(struct inode *inode,
  197. struct iomap *iomap, loff_t pos)
  198. {
  199. return iomap->type != IOMAP_MAPPED ||
  200. (iomap->flags & IOMAP_F_NEW) ||
  201. pos >= i_size_read(inode);
  202. }
  203. static loff_t
  204. iomap_readpage_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
  205. struct iomap *iomap, struct iomap *srcmap)
  206. {
  207. struct iomap_readpage_ctx *ctx = data;
  208. struct page *page = ctx->cur_page;
  209. struct iomap_page *iop = iomap_page_create(inode, page);
  210. bool same_page = false, is_contig = false;
  211. loff_t orig_pos = pos;
  212. unsigned poff, plen;
  213. sector_t sector;
  214. if (iomap->type == IOMAP_INLINE) {
  215. WARN_ON_ONCE(pos);
  216. iomap_read_inline_data(inode, page, iomap);
  217. return PAGE_SIZE;
  218. }
  219. /* zero post-eof blocks as the page may be mapped */
  220. iomap_adjust_read_range(inode, iop, &pos, length, &poff, &plen);
  221. if (plen == 0)
  222. goto done;
  223. if (iomap_block_needs_zeroing(inode, iomap, pos)) {
  224. zero_user(page, poff, plen);
  225. iomap_set_range_uptodate(page, poff, plen);
  226. goto done;
  227. }
  228. ctx->cur_page_in_bio = true;
  229. if (iop)
  230. atomic_add(plen, &iop->read_bytes_pending);
  231. /* Try to merge into a previous segment if we can */
  232. sector = iomap_sector(iomap, pos);
  233. if (ctx->bio && bio_end_sector(ctx->bio) == sector) {
  234. if (__bio_try_merge_page(ctx->bio, page, plen, poff,
  235. &same_page))
  236. goto done;
  237. is_contig = true;
  238. }
  239. if (!is_contig || bio_full(ctx->bio, plen)) {
  240. gfp_t gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL);
  241. gfp_t orig_gfp = gfp;
  242. int nr_vecs = (length + PAGE_SIZE - 1) >> PAGE_SHIFT;
  243. if (ctx->bio)
  244. submit_bio(ctx->bio);
  245. if (ctx->rac) /* same as readahead_gfp_mask */
  246. gfp |= __GFP_NORETRY | __GFP_NOWARN;
  247. ctx->bio = bio_alloc(gfp, min(BIO_MAX_PAGES, nr_vecs));
  248. /*
  249. * If the bio_alloc fails, try it again for a single page to
  250. * avoid having to deal with partial page reads. This emulates
  251. * what do_mpage_readpage does.
  252. */
  253. if (!ctx->bio)
  254. ctx->bio = bio_alloc(orig_gfp, 1);
  255. ctx->bio->bi_opf = REQ_OP_READ;
  256. if (ctx->rac)
  257. ctx->bio->bi_opf |= REQ_RAHEAD;
  258. ctx->bio->bi_iter.bi_sector = sector;
  259. bio_set_dev(ctx->bio, iomap->bdev);
  260. ctx->bio->bi_end_io = iomap_read_end_io;
  261. }
  262. bio_add_page(ctx->bio, page, plen, poff);
  263. done:
  264. /*
  265. * Move the caller beyond our range so that it keeps making progress.
  266. * For that we have to include any leading non-uptodate ranges, but
  267. * we can skip trailing ones as they will be handled in the next
  268. * iteration.
  269. */
  270. return pos - orig_pos + plen;
  271. }
  272. int
  273. iomap_readpage(struct page *page, const struct iomap_ops *ops)
  274. {
  275. struct iomap_readpage_ctx ctx = { .cur_page = page };
  276. struct inode *inode = page->mapping->host;
  277. unsigned poff;
  278. loff_t ret;
  279. trace_iomap_readpage(page->mapping->host, 1);
  280. for (poff = 0; poff < PAGE_SIZE; poff += ret) {
  281. ret = iomap_apply(inode, page_offset(page) + poff,
  282. PAGE_SIZE - poff, 0, ops, &ctx,
  283. iomap_readpage_actor);
  284. if (ret <= 0) {
  285. WARN_ON_ONCE(ret == 0);
  286. SetPageError(page);
  287. break;
  288. }
  289. }
  290. if (ctx.bio) {
  291. submit_bio(ctx.bio);
  292. WARN_ON_ONCE(!ctx.cur_page_in_bio);
  293. } else {
  294. WARN_ON_ONCE(ctx.cur_page_in_bio);
  295. unlock_page(page);
  296. }
  297. /*
  298. * Just like mpage_readahead and block_read_full_page we always
  299. * return 0 and just mark the page as PageError on errors. This
  300. * should be cleaned up all through the stack eventually.
  301. */
  302. return 0;
  303. }
  304. EXPORT_SYMBOL_GPL(iomap_readpage);
  305. static loff_t
  306. iomap_readahead_actor(struct inode *inode, loff_t pos, loff_t length,
  307. void *data, struct iomap *iomap, struct iomap *srcmap)
  308. {
  309. struct iomap_readpage_ctx *ctx = data;
  310. loff_t done, ret;
  311. for (done = 0; done < length; done += ret) {
  312. if (ctx->cur_page && offset_in_page(pos + done) == 0) {
  313. if (!ctx->cur_page_in_bio)
  314. unlock_page(ctx->cur_page);
  315. put_page(ctx->cur_page);
  316. ctx->cur_page = NULL;
  317. }
  318. if (!ctx->cur_page) {
  319. ctx->cur_page = readahead_page(ctx->rac);
  320. ctx->cur_page_in_bio = false;
  321. }
  322. ret = iomap_readpage_actor(inode, pos + done, length - done,
  323. ctx, iomap, srcmap);
  324. }
  325. return done;
  326. }
  327. /**
  328. * iomap_readahead - Attempt to read pages from a file.
  329. * @rac: Describes the pages to be read.
  330. * @ops: The operations vector for the filesystem.
  331. *
  332. * This function is for filesystems to call to implement their readahead
  333. * address_space operation.
  334. *
  335. * Context: The @ops callbacks may submit I/O (eg to read the addresses of
  336. * blocks from disc), and may wait for it. The caller may be trying to
  337. * access a different page, and so sleeping excessively should be avoided.
  338. * It may allocate memory, but should avoid costly allocations. This
  339. * function is called with memalloc_nofs set, so allocations will not cause
  340. * the filesystem to be reentered.
  341. */
  342. void iomap_readahead(struct readahead_control *rac, const struct iomap_ops *ops)
  343. {
  344. struct inode *inode = rac->mapping->host;
  345. loff_t pos = readahead_pos(rac);
  346. loff_t length = readahead_length(rac);
  347. struct iomap_readpage_ctx ctx = {
  348. .rac = rac,
  349. };
  350. trace_iomap_readahead(inode, readahead_count(rac));
  351. while (length > 0) {
  352. loff_t ret = iomap_apply(inode, pos, length, 0, ops,
  353. &ctx, iomap_readahead_actor);
  354. if (ret <= 0) {
  355. WARN_ON_ONCE(ret == 0);
  356. break;
  357. }
  358. pos += ret;
  359. length -= ret;
  360. }
  361. if (ctx.bio)
  362. submit_bio(ctx.bio);
  363. if (ctx.cur_page) {
  364. if (!ctx.cur_page_in_bio)
  365. unlock_page(ctx.cur_page);
  366. put_page(ctx.cur_page);
  367. }
  368. }
  369. EXPORT_SYMBOL_GPL(iomap_readahead);
  370. /*
  371. * iomap_is_partially_uptodate checks whether blocks within a page are
  372. * uptodate or not.
  373. *
  374. * Returns true if all blocks which correspond to a file portion
  375. * we want to read within the page are uptodate.
  376. */
  377. int
  378. iomap_is_partially_uptodate(struct page *page, unsigned long from,
  379. unsigned long count)
  380. {
  381. struct iomap_page *iop = to_iomap_page(page);
  382. struct inode *inode = page->mapping->host;
  383. unsigned len, first, last;
  384. unsigned i;
  385. /* Limit range to one page */
  386. len = min_t(unsigned, PAGE_SIZE - from, count);
  387. /* First and last blocks in range within page */
  388. first = from >> inode->i_blkbits;
  389. last = (from + len - 1) >> inode->i_blkbits;
  390. if (iop) {
  391. for (i = first; i <= last; i++)
  392. if (!test_bit(i, iop->uptodate))
  393. return 0;
  394. return 1;
  395. }
  396. return 0;
  397. }
  398. EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate);
  399. int
  400. iomap_releasepage(struct page *page, gfp_t gfp_mask)
  401. {
  402. trace_iomap_releasepage(page->mapping->host, page_offset(page),
  403. PAGE_SIZE);
  404. /*
  405. * mm accommodates an old ext3 case where clean pages might not have had
  406. * the dirty bit cleared. Thus, it can send actual dirty pages to
  407. * ->releasepage() via shrink_active_list(), skip those here.
  408. */
  409. if (PageDirty(page) || PageWriteback(page))
  410. return 0;
  411. iomap_page_release(page);
  412. return 1;
  413. }
  414. EXPORT_SYMBOL_GPL(iomap_releasepage);
  415. void
  416. iomap_invalidatepage(struct page *page, unsigned int offset, unsigned int len)
  417. {
  418. trace_iomap_invalidatepage(page->mapping->host, offset, len);
  419. /*
  420. * If we are invalidating the entire page, clear the dirty state from it
  421. * and release it to avoid unnecessary buildup of the LRU.
  422. */
  423. if (offset == 0 && len == PAGE_SIZE) {
  424. WARN_ON_ONCE(PageWriteback(page));
  425. cancel_dirty_page(page);
  426. iomap_page_release(page);
  427. }
  428. }
  429. EXPORT_SYMBOL_GPL(iomap_invalidatepage);
  430. #ifdef CONFIG_MIGRATION
  431. int
  432. iomap_migrate_page(struct address_space *mapping, struct page *newpage,
  433. struct page *page, enum migrate_mode mode)
  434. {
  435. int ret;
  436. ret = migrate_page_move_mapping(mapping, newpage, page, 0);
  437. if (ret != MIGRATEPAGE_SUCCESS)
  438. return ret;
  439. if (page_has_private(page))
  440. attach_page_private(newpage, detach_page_private(page));
  441. if (mode != MIGRATE_SYNC_NO_COPY)
  442. migrate_page_copy(newpage, page);
  443. else
  444. migrate_page_states(newpage, page);
  445. return MIGRATEPAGE_SUCCESS;
  446. }
  447. EXPORT_SYMBOL_GPL(iomap_migrate_page);
  448. #endif /* CONFIG_MIGRATION */
  449. enum {
  450. IOMAP_WRITE_F_UNSHARE = (1 << 0),
  451. };
  452. static void
  453. iomap_write_failed(struct inode *inode, loff_t pos, unsigned len)
  454. {
  455. loff_t i_size = i_size_read(inode);
  456. /*
  457. * Only truncate newly allocated pages beyoned EOF, even if the
  458. * write started inside the existing inode size.
  459. */
  460. if (pos + len > i_size)
  461. truncate_pagecache_range(inode, max(pos, i_size), pos + len);
  462. }
  463. static int
  464. iomap_read_page_sync(loff_t block_start, struct page *page, unsigned poff,
  465. unsigned plen, struct iomap *iomap)
  466. {
  467. struct bio_vec bvec;
  468. struct bio bio;
  469. bio_init(&bio, &bvec, 1);
  470. bio.bi_opf = REQ_OP_READ;
  471. bio.bi_iter.bi_sector = iomap_sector(iomap, block_start);
  472. bio_set_dev(&bio, iomap->bdev);
  473. __bio_add_page(&bio, page, plen, poff);
  474. return submit_bio_wait(&bio);
  475. }
  476. static int
  477. __iomap_write_begin(struct inode *inode, loff_t pos, unsigned len, int flags,
  478. struct page *page, struct iomap *srcmap)
  479. {
  480. struct iomap_page *iop = iomap_page_create(inode, page);
  481. loff_t block_size = i_blocksize(inode);
  482. loff_t block_start = round_down(pos, block_size);
  483. loff_t block_end = round_up(pos + len, block_size);
  484. unsigned from = offset_in_page(pos), to = from + len, poff, plen;
  485. if (PageUptodate(page))
  486. return 0;
  487. ClearPageError(page);
  488. do {
  489. iomap_adjust_read_range(inode, iop, &block_start,
  490. block_end - block_start, &poff, &plen);
  491. if (plen == 0)
  492. break;
  493. if (!(flags & IOMAP_WRITE_F_UNSHARE) &&
  494. (from <= poff || from >= poff + plen) &&
  495. (to <= poff || to >= poff + plen))
  496. continue;
  497. if (iomap_block_needs_zeroing(inode, srcmap, block_start)) {
  498. if (WARN_ON_ONCE(flags & IOMAP_WRITE_F_UNSHARE))
  499. return -EIO;
  500. zero_user_segments(page, poff, from, to, poff + plen);
  501. } else {
  502. int status = iomap_read_page_sync(block_start, page,
  503. poff, plen, srcmap);
  504. if (status)
  505. return status;
  506. }
  507. iomap_set_range_uptodate(page, poff, plen);
  508. } while ((block_start += plen) < block_end);
  509. return 0;
  510. }
  511. static int
  512. iomap_write_begin(struct inode *inode, loff_t pos, unsigned len, unsigned flags,
  513. struct page **pagep, struct iomap *iomap, struct iomap *srcmap)
  514. {
  515. const struct iomap_page_ops *page_ops = iomap->page_ops;
  516. struct page *page;
  517. int status = 0;
  518. BUG_ON(pos + len > iomap->offset + iomap->length);
  519. if (srcmap != iomap)
  520. BUG_ON(pos + len > srcmap->offset + srcmap->length);
  521. if (fatal_signal_pending(current))
  522. return -EINTR;
  523. if (page_ops && page_ops->page_prepare) {
  524. status = page_ops->page_prepare(inode, pos, len, iomap);
  525. if (status)
  526. return status;
  527. }
  528. page = grab_cache_page_write_begin(inode->i_mapping, pos >> PAGE_SHIFT,
  529. AOP_FLAG_NOFS);
  530. if (!page) {
  531. status = -ENOMEM;
  532. goto out_no_page;
  533. }
  534. if (srcmap->type == IOMAP_INLINE)
  535. iomap_read_inline_data(inode, page, srcmap);
  536. else if (iomap->flags & IOMAP_F_BUFFER_HEAD)
  537. status = __block_write_begin_int(page, pos, len, NULL, srcmap);
  538. else
  539. status = __iomap_write_begin(inode, pos, len, flags, page,
  540. srcmap);
  541. if (unlikely(status))
  542. goto out_unlock;
  543. *pagep = page;
  544. return 0;
  545. out_unlock:
  546. unlock_page(page);
  547. put_page(page);
  548. iomap_write_failed(inode, pos, len);
  549. out_no_page:
  550. if (page_ops && page_ops->page_done)
  551. page_ops->page_done(inode, pos, 0, NULL, iomap);
  552. return status;
  553. }
  554. int
  555. iomap_set_page_dirty(struct page *page)
  556. {
  557. struct address_space *mapping = page_mapping(page);
  558. int newly_dirty;
  559. if (unlikely(!mapping))
  560. return !TestSetPageDirty(page);
  561. /*
  562. * Lock out page->mem_cgroup migration to keep PageDirty
  563. * synchronized with per-memcg dirty page counters.
  564. */
  565. lock_page_memcg(page);
  566. newly_dirty = !TestSetPageDirty(page);
  567. if (newly_dirty)
  568. __set_page_dirty(page, mapping, 0);
  569. unlock_page_memcg(page);
  570. if (newly_dirty)
  571. __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
  572. return newly_dirty;
  573. }
  574. EXPORT_SYMBOL_GPL(iomap_set_page_dirty);
  575. static size_t __iomap_write_end(struct inode *inode, loff_t pos, size_t len,
  576. size_t copied, struct page *page)
  577. {
  578. flush_dcache_page(page);
  579. /*
  580. * The blocks that were entirely written will now be uptodate, so we
  581. * don't have to worry about a readpage reading them and overwriting a
  582. * partial write. However if we have encountered a short write and only
  583. * partially written into a block, it will not be marked uptodate, so a
  584. * readpage might come in and destroy our partial write.
  585. *
  586. * Do the simplest thing, and just treat any short write to a non
  587. * uptodate page as a zero-length write, and force the caller to redo
  588. * the whole thing.
  589. */
  590. if (unlikely(copied < len && !PageUptodate(page)))
  591. return 0;
  592. iomap_set_range_uptodate(page, offset_in_page(pos), len);
  593. iomap_set_page_dirty(page);
  594. return copied;
  595. }
  596. static size_t iomap_write_end_inline(struct inode *inode, struct page *page,
  597. struct iomap *iomap, loff_t pos, size_t copied)
  598. {
  599. void *addr;
  600. WARN_ON_ONCE(!PageUptodate(page));
  601. BUG_ON(pos + copied > PAGE_SIZE - offset_in_page(iomap->inline_data));
  602. flush_dcache_page(page);
  603. addr = kmap_atomic(page);
  604. memcpy(iomap->inline_data + pos, addr + pos, copied);
  605. kunmap_atomic(addr);
  606. mark_inode_dirty(inode);
  607. return copied;
  608. }
  609. /* Returns the number of bytes copied. May be 0. Cannot be an errno. */
  610. static size_t iomap_write_end(struct inode *inode, loff_t pos, size_t len,
  611. size_t copied, struct page *page, struct iomap *iomap,
  612. struct iomap *srcmap)
  613. {
  614. const struct iomap_page_ops *page_ops = iomap->page_ops;
  615. loff_t old_size = inode->i_size;
  616. size_t ret;
  617. if (srcmap->type == IOMAP_INLINE) {
  618. ret = iomap_write_end_inline(inode, page, iomap, pos, copied);
  619. } else if (srcmap->flags & IOMAP_F_BUFFER_HEAD) {
  620. ret = block_write_end(NULL, inode->i_mapping, pos, len, copied,
  621. page, NULL);
  622. } else {
  623. ret = __iomap_write_end(inode, pos, len, copied, page);
  624. }
  625. /*
  626. * Update the in-memory inode size after copying the data into the page
  627. * cache. It's up to the file system to write the updated size to disk,
  628. * preferably after I/O completion so that no stale data is exposed.
  629. */
  630. if (pos + ret > old_size) {
  631. i_size_write(inode, pos + ret);
  632. iomap->flags |= IOMAP_F_SIZE_CHANGED;
  633. }
  634. unlock_page(page);
  635. if (old_size < pos)
  636. pagecache_isize_extended(inode, old_size, pos);
  637. if (page_ops && page_ops->page_done)
  638. page_ops->page_done(inode, pos, ret, page, iomap);
  639. put_page(page);
  640. if (ret < len)
  641. iomap_write_failed(inode, pos, len);
  642. return ret;
  643. }
  644. static loff_t
  645. iomap_write_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
  646. struct iomap *iomap, struct iomap *srcmap)
  647. {
  648. struct iov_iter *i = data;
  649. long status = 0;
  650. ssize_t written = 0;
  651. do {
  652. struct page *page;
  653. unsigned long offset; /* Offset into pagecache page */
  654. unsigned long bytes; /* Bytes to write to page */
  655. size_t copied; /* Bytes copied from user */
  656. offset = offset_in_page(pos);
  657. bytes = min_t(unsigned long, PAGE_SIZE - offset,
  658. iov_iter_count(i));
  659. again:
  660. if (bytes > length)
  661. bytes = length;
  662. /*
  663. * Bring in the user page that we will copy from _first_.
  664. * Otherwise there's a nasty deadlock on copying from the
  665. * same page as we're writing to, without it being marked
  666. * up-to-date.
  667. *
  668. * Not only is this an optimisation, but it is also required
  669. * to check that the address is actually valid, when atomic
  670. * usercopies are used, below.
  671. */
  672. if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
  673. status = -EFAULT;
  674. break;
  675. }
  676. status = iomap_write_begin(inode, pos, bytes, 0, &page, iomap,
  677. srcmap);
  678. if (unlikely(status))
  679. break;
  680. if (mapping_writably_mapped(inode->i_mapping))
  681. flush_dcache_page(page);
  682. copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
  683. copied = iomap_write_end(inode, pos, bytes, copied, page, iomap,
  684. srcmap);
  685. cond_resched();
  686. iov_iter_advance(i, copied);
  687. if (unlikely(copied == 0)) {
  688. /*
  689. * If we were unable to copy any data at all, we must
  690. * fall back to a single segment length write.
  691. *
  692. * If we didn't fallback here, we could livelock
  693. * because not all segments in the iov can be copied at
  694. * once without a pagefault.
  695. */
  696. bytes = min_t(unsigned long, PAGE_SIZE - offset,
  697. iov_iter_single_seg_count(i));
  698. goto again;
  699. }
  700. pos += copied;
  701. written += copied;
  702. length -= copied;
  703. balance_dirty_pages_ratelimited(inode->i_mapping);
  704. } while (iov_iter_count(i) && length);
  705. return written ? written : status;
  706. }
  707. ssize_t
  708. iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *iter,
  709. const struct iomap_ops *ops)
  710. {
  711. struct inode *inode = iocb->ki_filp->f_mapping->host;
  712. loff_t pos = iocb->ki_pos, ret = 0, written = 0;
  713. while (iov_iter_count(iter)) {
  714. ret = iomap_apply(inode, pos, iov_iter_count(iter),
  715. IOMAP_WRITE, ops, iter, iomap_write_actor);
  716. if (ret <= 0)
  717. break;
  718. pos += ret;
  719. written += ret;
  720. }
  721. return written ? written : ret;
  722. }
  723. EXPORT_SYMBOL_GPL(iomap_file_buffered_write);
  724. static loff_t
  725. iomap_unshare_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
  726. struct iomap *iomap, struct iomap *srcmap)
  727. {
  728. long status = 0;
  729. loff_t written = 0;
  730. /* don't bother with blocks that are not shared to start with */
  731. if (!(iomap->flags & IOMAP_F_SHARED))
  732. return length;
  733. /* don't bother with holes or unwritten extents */
  734. if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN)
  735. return length;
  736. do {
  737. unsigned long offset = offset_in_page(pos);
  738. unsigned long bytes = min_t(loff_t, PAGE_SIZE - offset, length);
  739. struct page *page;
  740. status = iomap_write_begin(inode, pos, bytes,
  741. IOMAP_WRITE_F_UNSHARE, &page, iomap, srcmap);
  742. if (unlikely(status))
  743. return status;
  744. status = iomap_write_end(inode, pos, bytes, bytes, page, iomap,
  745. srcmap);
  746. if (WARN_ON_ONCE(status == 0))
  747. return -EIO;
  748. cond_resched();
  749. pos += status;
  750. written += status;
  751. length -= status;
  752. balance_dirty_pages_ratelimited(inode->i_mapping);
  753. } while (length);
  754. return written;
  755. }
  756. int
  757. iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len,
  758. const struct iomap_ops *ops)
  759. {
  760. loff_t ret;
  761. while (len) {
  762. ret = iomap_apply(inode, pos, len, IOMAP_WRITE, ops, NULL,
  763. iomap_unshare_actor);
  764. if (ret <= 0)
  765. return ret;
  766. pos += ret;
  767. len -= ret;
  768. }
  769. return 0;
  770. }
  771. EXPORT_SYMBOL_GPL(iomap_file_unshare);
  772. static s64 iomap_zero(struct inode *inode, loff_t pos, u64 length,
  773. struct iomap *iomap, struct iomap *srcmap)
  774. {
  775. struct page *page;
  776. int status;
  777. unsigned offset = offset_in_page(pos);
  778. unsigned bytes = min_t(u64, PAGE_SIZE - offset, length);
  779. status = iomap_write_begin(inode, pos, bytes, 0, &page, iomap, srcmap);
  780. if (status)
  781. return status;
  782. zero_user(page, offset, bytes);
  783. mark_page_accessed(page);
  784. return iomap_write_end(inode, pos, bytes, bytes, page, iomap, srcmap);
  785. }
  786. static loff_t iomap_zero_range_actor(struct inode *inode, loff_t pos,
  787. loff_t length, void *data, struct iomap *iomap,
  788. struct iomap *srcmap)
  789. {
  790. bool *did_zero = data;
  791. loff_t written = 0;
  792. /* already zeroed? we're done. */
  793. if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN)
  794. return length;
  795. do {
  796. s64 bytes;
  797. if (IS_DAX(inode))
  798. bytes = dax_iomap_zero(pos, length, iomap);
  799. else
  800. bytes = iomap_zero(inode, pos, length, iomap, srcmap);
  801. if (bytes < 0)
  802. return bytes;
  803. pos += bytes;
  804. length -= bytes;
  805. written += bytes;
  806. if (did_zero)
  807. *did_zero = true;
  808. } while (length > 0);
  809. return written;
  810. }
  811. int
  812. iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
  813. const struct iomap_ops *ops)
  814. {
  815. loff_t ret;
  816. while (len > 0) {
  817. ret = iomap_apply(inode, pos, len, IOMAP_ZERO,
  818. ops, did_zero, iomap_zero_range_actor);
  819. if (ret <= 0)
  820. return ret;
  821. pos += ret;
  822. len -= ret;
  823. }
  824. return 0;
  825. }
  826. EXPORT_SYMBOL_GPL(iomap_zero_range);
  827. int
  828. iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
  829. const struct iomap_ops *ops)
  830. {
  831. unsigned int blocksize = i_blocksize(inode);
  832. unsigned int off = pos & (blocksize - 1);
  833. /* Block boundary? Nothing to do */
  834. if (!off)
  835. return 0;
  836. return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops);
  837. }
  838. EXPORT_SYMBOL_GPL(iomap_truncate_page);
  839. static loff_t
  840. iomap_page_mkwrite_actor(struct inode *inode, loff_t pos, loff_t length,
  841. void *data, struct iomap *iomap, struct iomap *srcmap)
  842. {
  843. struct page *page = data;
  844. int ret;
  845. if (iomap->flags & IOMAP_F_BUFFER_HEAD) {
  846. ret = __block_write_begin_int(page, pos, length, NULL, iomap);
  847. if (ret)
  848. return ret;
  849. block_commit_write(page, 0, length);
  850. } else {
  851. WARN_ON_ONCE(!PageUptodate(page));
  852. iomap_page_create(inode, page);
  853. set_page_dirty(page);
  854. }
  855. return length;
  856. }
  857. vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops)
  858. {
  859. struct page *page = vmf->page;
  860. struct inode *inode = file_inode(vmf->vma->vm_file);
  861. unsigned long length;
  862. loff_t offset;
  863. ssize_t ret;
  864. lock_page(page);
  865. ret = page_mkwrite_check_truncate(page, inode);
  866. if (ret < 0)
  867. goto out_unlock;
  868. length = ret;
  869. offset = page_offset(page);
  870. while (length > 0) {
  871. ret = iomap_apply(inode, offset, length,
  872. IOMAP_WRITE | IOMAP_FAULT, ops, page,
  873. iomap_page_mkwrite_actor);
  874. if (unlikely(ret <= 0))
  875. goto out_unlock;
  876. offset += ret;
  877. length -= ret;
  878. }
  879. wait_for_stable_page(page);
  880. return VM_FAULT_LOCKED;
  881. out_unlock:
  882. unlock_page(page);
  883. return block_page_mkwrite_return(ret);
  884. }
  885. EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
  886. static void
  887. iomap_finish_page_writeback(struct inode *inode, struct page *page,
  888. int error, unsigned int len)
  889. {
  890. struct iomap_page *iop = to_iomap_page(page);
  891. if (error) {
  892. SetPageError(page);
  893. mapping_set_error(inode->i_mapping, error);
  894. }
  895. WARN_ON_ONCE(i_blocks_per_page(inode, page) > 1 && !iop);
  896. WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) <= 0);
  897. if (!iop || atomic_sub_and_test(len, &iop->write_bytes_pending))
  898. end_page_writeback(page);
  899. }
  900. /*
  901. * We're now finished for good with this ioend structure. Update the page
  902. * state, release holds on bios, and finally free up memory. Do not use the
  903. * ioend after this.
  904. */
  905. static void
  906. iomap_finish_ioend(struct iomap_ioend *ioend, int error)
  907. {
  908. struct inode *inode = ioend->io_inode;
  909. struct bio *bio = &ioend->io_inline_bio;
  910. struct bio *last = ioend->io_bio, *next;
  911. u64 start = bio->bi_iter.bi_sector;
  912. loff_t offset = ioend->io_offset;
  913. bool quiet = bio_flagged(bio, BIO_QUIET);
  914. for (bio = &ioend->io_inline_bio; bio; bio = next) {
  915. struct bio_vec *bv;
  916. struct bvec_iter_all iter_all;
  917. /*
  918. * For the last bio, bi_private points to the ioend, so we
  919. * need to explicitly end the iteration here.
  920. */
  921. if (bio == last)
  922. next = NULL;
  923. else
  924. next = bio->bi_private;
  925. /* walk each page on bio, ending page IO on them */
  926. bio_for_each_segment_all(bv, bio, iter_all)
  927. iomap_finish_page_writeback(inode, bv->bv_page, error,
  928. bv->bv_len);
  929. bio_put(bio);
  930. }
  931. /* The ioend has been freed by bio_put() */
  932. if (unlikely(error && !quiet)) {
  933. printk_ratelimited(KERN_ERR
  934. "%s: writeback error on inode %lu, offset %lld, sector %llu",
  935. inode->i_sb->s_id, inode->i_ino, offset, start);
  936. }
  937. }
  938. void
  939. iomap_finish_ioends(struct iomap_ioend *ioend, int error)
  940. {
  941. struct list_head tmp;
  942. list_replace_init(&ioend->io_list, &tmp);
  943. iomap_finish_ioend(ioend, error);
  944. while (!list_empty(&tmp)) {
  945. ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
  946. list_del_init(&ioend->io_list);
  947. iomap_finish_ioend(ioend, error);
  948. }
  949. }
  950. EXPORT_SYMBOL_GPL(iomap_finish_ioends);
  951. /*
  952. * We can merge two adjacent ioends if they have the same set of work to do.
  953. */
  954. static bool
  955. iomap_ioend_can_merge(struct iomap_ioend *ioend, struct iomap_ioend *next)
  956. {
  957. if (ioend->io_bio->bi_status != next->io_bio->bi_status)
  958. return false;
  959. if ((ioend->io_flags & IOMAP_F_SHARED) ^
  960. (next->io_flags & IOMAP_F_SHARED))
  961. return false;
  962. if ((ioend->io_type == IOMAP_UNWRITTEN) ^
  963. (next->io_type == IOMAP_UNWRITTEN))
  964. return false;
  965. if (ioend->io_offset + ioend->io_size != next->io_offset)
  966. return false;
  967. return true;
  968. }
  969. void
  970. iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends,
  971. void (*merge_private)(struct iomap_ioend *ioend,
  972. struct iomap_ioend *next))
  973. {
  974. struct iomap_ioend *next;
  975. INIT_LIST_HEAD(&ioend->io_list);
  976. while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
  977. io_list))) {
  978. if (!iomap_ioend_can_merge(ioend, next))
  979. break;
  980. list_move_tail(&next->io_list, &ioend->io_list);
  981. ioend->io_size += next->io_size;
  982. if (next->io_private && merge_private)
  983. merge_private(ioend, next);
  984. }
  985. }
  986. EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
  987. static int
  988. iomap_ioend_compare(void *priv, struct list_head *a, struct list_head *b)
  989. {
  990. struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list);
  991. struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list);
  992. if (ia->io_offset < ib->io_offset)
  993. return -1;
  994. if (ia->io_offset > ib->io_offset)
  995. return 1;
  996. return 0;
  997. }
  998. void
  999. iomap_sort_ioends(struct list_head *ioend_list)
  1000. {
  1001. list_sort(NULL, ioend_list, iomap_ioend_compare);
  1002. }
  1003. EXPORT_SYMBOL_GPL(iomap_sort_ioends);
  1004. static void iomap_writepage_end_bio(struct bio *bio)
  1005. {
  1006. struct iomap_ioend *ioend = bio->bi_private;
  1007. iomap_finish_ioend(ioend, blk_status_to_errno(bio->bi_status));
  1008. }
  1009. /*
  1010. * Submit the final bio for an ioend.
  1011. *
  1012. * If @error is non-zero, it means that we have a situation where some part of
  1013. * the submission process has failed after we have marked paged for writeback
  1014. * and unlocked them. In this situation, we need to fail the bio instead of
  1015. * submitting it. This typically only happens on a filesystem shutdown.
  1016. */
  1017. static int
  1018. iomap_submit_ioend(struct iomap_writepage_ctx *wpc, struct iomap_ioend *ioend,
  1019. int error)
  1020. {
  1021. ioend->io_bio->bi_private = ioend;
  1022. ioend->io_bio->bi_end_io = iomap_writepage_end_bio;
  1023. if (wpc->ops->prepare_ioend)
  1024. error = wpc->ops->prepare_ioend(ioend, error);
  1025. if (error) {
  1026. /*
  1027. * If we are failing the IO now, just mark the ioend with an
  1028. * error and finish it. This will run IO completion immediately
  1029. * as there is only one reference to the ioend at this point in
  1030. * time.
  1031. */
  1032. ioend->io_bio->bi_status = errno_to_blk_status(error);
  1033. bio_endio(ioend->io_bio);
  1034. return error;
  1035. }
  1036. submit_bio(ioend->io_bio);
  1037. return 0;
  1038. }
  1039. static struct iomap_ioend *
  1040. iomap_alloc_ioend(struct inode *inode, struct iomap_writepage_ctx *wpc,
  1041. loff_t offset, sector_t sector, struct writeback_control *wbc)
  1042. {
  1043. struct iomap_ioend *ioend;
  1044. struct bio *bio;
  1045. bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &iomap_ioend_bioset);
  1046. bio_set_dev(bio, wpc->iomap.bdev);
  1047. bio->bi_iter.bi_sector = sector;
  1048. bio->bi_opf = REQ_OP_WRITE | wbc_to_write_flags(wbc);
  1049. bio->bi_write_hint = inode->i_write_hint;
  1050. wbc_init_bio(wbc, bio);
  1051. ioend = container_of(bio, struct iomap_ioend, io_inline_bio);
  1052. INIT_LIST_HEAD(&ioend->io_list);
  1053. ioend->io_type = wpc->iomap.type;
  1054. ioend->io_flags = wpc->iomap.flags;
  1055. ioend->io_inode = inode;
  1056. ioend->io_size = 0;
  1057. ioend->io_offset = offset;
  1058. ioend->io_private = NULL;
  1059. ioend->io_bio = bio;
  1060. return ioend;
  1061. }
  1062. /*
  1063. * Allocate a new bio, and chain the old bio to the new one.
  1064. *
  1065. * Note that we have to do perform the chaining in this unintuitive order
  1066. * so that the bi_private linkage is set up in the right direction for the
  1067. * traversal in iomap_finish_ioend().
  1068. */
  1069. static struct bio *
  1070. iomap_chain_bio(struct bio *prev)
  1071. {
  1072. struct bio *new;
  1073. new = bio_alloc(GFP_NOFS, BIO_MAX_PAGES);
  1074. bio_copy_dev(new, prev);/* also copies over blkcg information */
  1075. new->bi_iter.bi_sector = bio_end_sector(prev);
  1076. new->bi_opf = prev->bi_opf;
  1077. new->bi_write_hint = prev->bi_write_hint;
  1078. bio_chain(prev, new);
  1079. bio_get(prev); /* for iomap_finish_ioend */
  1080. submit_bio(prev);
  1081. return new;
  1082. }
  1083. static bool
  1084. iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t offset,
  1085. sector_t sector)
  1086. {
  1087. if ((wpc->iomap.flags & IOMAP_F_SHARED) !=
  1088. (wpc->ioend->io_flags & IOMAP_F_SHARED))
  1089. return false;
  1090. if (wpc->iomap.type != wpc->ioend->io_type)
  1091. return false;
  1092. if (offset != wpc->ioend->io_offset + wpc->ioend->io_size)
  1093. return false;
  1094. if (sector != bio_end_sector(wpc->ioend->io_bio))
  1095. return false;
  1096. return true;
  1097. }
  1098. /*
  1099. * Test to see if we have an existing ioend structure that we could append to
  1100. * first, otherwise finish off the current ioend and start another.
  1101. */
  1102. static void
  1103. iomap_add_to_ioend(struct inode *inode, loff_t offset, struct page *page,
  1104. struct iomap_page *iop, struct iomap_writepage_ctx *wpc,
  1105. struct writeback_control *wbc, struct list_head *iolist)
  1106. {
  1107. sector_t sector = iomap_sector(&wpc->iomap, offset);
  1108. unsigned len = i_blocksize(inode);
  1109. unsigned poff = offset & (PAGE_SIZE - 1);
  1110. bool merged, same_page = false;
  1111. if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, offset, sector)) {
  1112. if (wpc->ioend)
  1113. list_add(&wpc->ioend->io_list, iolist);
  1114. wpc->ioend = iomap_alloc_ioend(inode, wpc, offset, sector, wbc);
  1115. }
  1116. merged = __bio_try_merge_page(wpc->ioend->io_bio, page, len, poff,
  1117. &same_page);
  1118. if (iop)
  1119. atomic_add(len, &iop->write_bytes_pending);
  1120. if (!merged) {
  1121. if (bio_full(wpc->ioend->io_bio, len)) {
  1122. wpc->ioend->io_bio =
  1123. iomap_chain_bio(wpc->ioend->io_bio);
  1124. }
  1125. bio_add_page(wpc->ioend->io_bio, page, len, poff);
  1126. }
  1127. wpc->ioend->io_size += len;
  1128. wbc_account_cgroup_owner(wbc, page, len);
  1129. }
  1130. /*
  1131. * We implement an immediate ioend submission policy here to avoid needing to
  1132. * chain multiple ioends and hence nest mempool allocations which can violate
  1133. * forward progress guarantees we need to provide. The current ioend we are
  1134. * adding blocks to is cached on the writepage context, and if the new block
  1135. * does not append to the cached ioend it will create a new ioend and cache that
  1136. * instead.
  1137. *
  1138. * If a new ioend is created and cached, the old ioend is returned and queued
  1139. * locally for submission once the entire page is processed or an error has been
  1140. * detected. While ioends are submitted immediately after they are completed,
  1141. * batching optimisations are provided by higher level block plugging.
  1142. *
  1143. * At the end of a writeback pass, there will be a cached ioend remaining on the
  1144. * writepage context that the caller will need to submit.
  1145. */
  1146. static int
  1147. iomap_writepage_map(struct iomap_writepage_ctx *wpc,
  1148. struct writeback_control *wbc, struct inode *inode,
  1149. struct page *page, u64 end_offset)
  1150. {
  1151. struct iomap_page *iop = to_iomap_page(page);
  1152. struct iomap_ioend *ioend, *next;
  1153. unsigned len = i_blocksize(inode);
  1154. u64 file_offset; /* file offset of page */
  1155. int error = 0, count = 0, i;
  1156. LIST_HEAD(submit_list);
  1157. WARN_ON_ONCE(i_blocks_per_page(inode, page) > 1 && !iop);
  1158. WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) != 0);
  1159. /*
  1160. * Walk through the page to find areas to write back. If we run off the
  1161. * end of the current map or find the current map invalid, grab a new
  1162. * one.
  1163. */
  1164. for (i = 0, file_offset = page_offset(page);
  1165. i < (PAGE_SIZE >> inode->i_blkbits) && file_offset < end_offset;
  1166. i++, file_offset += len) {
  1167. if (iop && !test_bit(i, iop->uptodate))
  1168. continue;
  1169. error = wpc->ops->map_blocks(wpc, inode, file_offset);
  1170. if (error)
  1171. break;
  1172. if (WARN_ON_ONCE(wpc->iomap.type == IOMAP_INLINE))
  1173. continue;
  1174. if (wpc->iomap.type == IOMAP_HOLE)
  1175. continue;
  1176. iomap_add_to_ioend(inode, file_offset, page, iop, wpc, wbc,
  1177. &submit_list);
  1178. count++;
  1179. }
  1180. WARN_ON_ONCE(!wpc->ioend && !list_empty(&submit_list));
  1181. WARN_ON_ONCE(!PageLocked(page));
  1182. WARN_ON_ONCE(PageWriteback(page));
  1183. WARN_ON_ONCE(PageDirty(page));
  1184. /*
  1185. * We cannot cancel the ioend directly here on error. We may have
  1186. * already set other pages under writeback and hence we have to run I/O
  1187. * completion to mark the error state of the pages under writeback
  1188. * appropriately.
  1189. */
  1190. if (unlikely(error)) {
  1191. /*
  1192. * Let the filesystem know what portion of the current page
  1193. * failed to map. If the page wasn't been added to ioend, it
  1194. * won't be affected by I/O completion and we must unlock it
  1195. * now.
  1196. */
  1197. if (wpc->ops->discard_page)
  1198. wpc->ops->discard_page(page, file_offset);
  1199. if (!count) {
  1200. ClearPageUptodate(page);
  1201. unlock_page(page);
  1202. goto done;
  1203. }
  1204. }
  1205. set_page_writeback(page);
  1206. unlock_page(page);
  1207. /*
  1208. * Preserve the original error if there was one, otherwise catch
  1209. * submission errors here and propagate into subsequent ioend
  1210. * submissions.
  1211. */
  1212. list_for_each_entry_safe(ioend, next, &submit_list, io_list) {
  1213. int error2;
  1214. list_del_init(&ioend->io_list);
  1215. error2 = iomap_submit_ioend(wpc, ioend, error);
  1216. if (error2 && !error)
  1217. error = error2;
  1218. }
  1219. /*
  1220. * We can end up here with no error and nothing to write only if we race
  1221. * with a partial page truncate on a sub-page block sized filesystem.
  1222. */
  1223. if (!count)
  1224. end_page_writeback(page);
  1225. done:
  1226. mapping_set_error(page->mapping, error);
  1227. return error;
  1228. }
  1229. /*
  1230. * Write out a dirty page.
  1231. *
  1232. * For delalloc space on the page we need to allocate space and flush it.
  1233. * For unwritten space on the page we need to start the conversion to
  1234. * regular allocated space.
  1235. */
  1236. static int
  1237. iomap_do_writepage(struct page *page, struct writeback_control *wbc, void *data)
  1238. {
  1239. struct iomap_writepage_ctx *wpc = data;
  1240. struct inode *inode = page->mapping->host;
  1241. pgoff_t end_index;
  1242. u64 end_offset;
  1243. loff_t offset;
  1244. trace_iomap_writepage(inode, page_offset(page), PAGE_SIZE);
  1245. /*
  1246. * Refuse to write the page out if we are called from reclaim context.
  1247. *
  1248. * This avoids stack overflows when called from deeply used stacks in
  1249. * random callers for direct reclaim or memcg reclaim. We explicitly
  1250. * allow reclaim from kswapd as the stack usage there is relatively low.
  1251. *
  1252. * This should never happen except in the case of a VM regression so
  1253. * warn about it.
  1254. */
  1255. if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) ==
  1256. PF_MEMALLOC))
  1257. goto redirty;
  1258. /*
  1259. * Given that we do not allow direct reclaim to call us, we should
  1260. * never be called in a recursive filesystem reclaim context.
  1261. */
  1262. if (WARN_ON_ONCE(current->flags & PF_MEMALLOC_NOFS))
  1263. goto redirty;
  1264. /*
  1265. * Is this page beyond the end of the file?
  1266. *
  1267. * The page index is less than the end_index, adjust the end_offset
  1268. * to the highest offset that this page should represent.
  1269. * -----------------------------------------------------
  1270. * | file mapping | <EOF> |
  1271. * -----------------------------------------------------
  1272. * | Page ... | Page N-2 | Page N-1 | Page N | |
  1273. * ^--------------------------------^----------|--------
  1274. * | desired writeback range | see else |
  1275. * ---------------------------------^------------------|
  1276. */
  1277. offset = i_size_read(inode);
  1278. end_index = offset >> PAGE_SHIFT;
  1279. if (page->index < end_index)
  1280. end_offset = (loff_t)(page->index + 1) << PAGE_SHIFT;
  1281. else {
  1282. /*
  1283. * Check whether the page to write out is beyond or straddles
  1284. * i_size or not.
  1285. * -------------------------------------------------------
  1286. * | file mapping | <EOF> |
  1287. * -------------------------------------------------------
  1288. * | Page ... | Page N-2 | Page N-1 | Page N | Beyond |
  1289. * ^--------------------------------^-----------|---------
  1290. * | | Straddles |
  1291. * ---------------------------------^-----------|--------|
  1292. */
  1293. unsigned offset_into_page = offset & (PAGE_SIZE - 1);
  1294. /*
  1295. * Skip the page if it is fully outside i_size, e.g. due to a
  1296. * truncate operation that is in progress. We must redirty the
  1297. * page so that reclaim stops reclaiming it. Otherwise
  1298. * iomap_vm_releasepage() is called on it and gets confused.
  1299. *
  1300. * Note that the end_index is unsigned long, it would overflow
  1301. * if the given offset is greater than 16TB on 32-bit system
  1302. * and if we do check the page is fully outside i_size or not
  1303. * via "if (page->index >= end_index + 1)" as "end_index + 1"
  1304. * will be evaluated to 0. Hence this page will be redirtied
  1305. * and be written out repeatedly which would result in an
  1306. * infinite loop, the user program that perform this operation
  1307. * will hang. Instead, we can verify this situation by checking
  1308. * if the page to write is totally beyond the i_size or if it's
  1309. * offset is just equal to the EOF.
  1310. */
  1311. if (page->index > end_index ||
  1312. (page->index == end_index && offset_into_page == 0))
  1313. goto redirty;
  1314. /*
  1315. * The page straddles i_size. It must be zeroed out on each
  1316. * and every writepage invocation because it may be mmapped.
  1317. * "A file is mapped in multiples of the page size. For a file
  1318. * that is not a multiple of the page size, the remaining
  1319. * memory is zeroed when mapped, and writes to that region are
  1320. * not written out to the file."
  1321. */
  1322. zero_user_segment(page, offset_into_page, PAGE_SIZE);
  1323. /* Adjust the end_offset to the end of file */
  1324. end_offset = offset;
  1325. }
  1326. return iomap_writepage_map(wpc, wbc, inode, page, end_offset);
  1327. redirty:
  1328. redirty_page_for_writepage(wbc, page);
  1329. unlock_page(page);
  1330. return 0;
  1331. }
  1332. int
  1333. iomap_writepage(struct page *page, struct writeback_control *wbc,
  1334. struct iomap_writepage_ctx *wpc,
  1335. const struct iomap_writeback_ops *ops)
  1336. {
  1337. int ret;
  1338. wpc->ops = ops;
  1339. ret = iomap_do_writepage(page, wbc, wpc);
  1340. if (!wpc->ioend)
  1341. return ret;
  1342. return iomap_submit_ioend(wpc, wpc->ioend, ret);
  1343. }
  1344. EXPORT_SYMBOL_GPL(iomap_writepage);
  1345. int
  1346. iomap_writepages(struct address_space *mapping, struct writeback_control *wbc,
  1347. struct iomap_writepage_ctx *wpc,
  1348. const struct iomap_writeback_ops *ops)
  1349. {
  1350. int ret;
  1351. wpc->ops = ops;
  1352. ret = write_cache_pages(mapping, wbc, iomap_do_writepage, wpc);
  1353. if (!wpc->ioend)
  1354. return ret;
  1355. return iomap_submit_ioend(wpc, wpc->ioend, ret);
  1356. }
  1357. EXPORT_SYMBOL_GPL(iomap_writepages);
  1358. static int __init iomap_init(void)
  1359. {
  1360. return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
  1361. offsetof(struct iomap_ioend, io_inline_bio),
  1362. BIOSET_NEED_BVECS);
  1363. }
  1364. fs_initcall(iomap_init);