mpage.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835
  1. /*
  2. * fs/mpage.c
  3. *
  4. * Copyright (C) 2002, Linus Torvalds.
  5. *
  6. * Contains functions related to preparing and submitting BIOs which contain
  7. * multiple pagecache pages.
  8. *
  9. * 15May2002 akpm@zip.com.au
  10. * Initial version
  11. * 27Jun2002 axboe@suse.de
  12. * use bio_add_page() to build bio's just the right size
  13. */
  14. #include <linux/kernel.h>
  15. #include <linux/module.h>
  16. #include <linux/mm.h>
  17. #include <linux/kdev_t.h>
  18. #include <linux/bio.h>
  19. #include <linux/fs.h>
  20. #include <linux/buffer_head.h>
  21. #include <linux/blkdev.h>
  22. #include <linux/highmem.h>
  23. #include <linux/prefetch.h>
  24. #include <linux/mpage.h>
  25. #include <linux/writeback.h>
  26. #include <linux/backing-dev.h>
  27. #include <linux/pagevec.h>
  28. /*
  29. * I/O completion handler for multipage BIOs.
  30. *
  31. * The mpage code never puts partial pages into a BIO (except for end-of-file).
  32. * If a page does not map to a contiguous run of blocks then it simply falls
  33. * back to block_read_full_page().
  34. *
  35. * Why is this? If a page's completion depends on a number of different BIOs
  36. * which can complete in any order (or at the same time) then determining the
  37. * status of that page is hard. See end_buffer_async_read() for the details.
  38. * There is no point in duplicating all that complexity.
  39. */
  40. static int mpage_end_io_read(struct bio *bio, unsigned int bytes_done, int err)
  41. {
  42. const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  43. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  44. if (bio->bi_size)
  45. return 1;
  46. do {
  47. struct page *page = bvec->bv_page;
  48. if (--bvec >= bio->bi_io_vec)
  49. prefetchw(&bvec->bv_page->flags);
  50. if (uptodate) {
  51. SetPageUptodate(page);
  52. } else {
  53. ClearPageUptodate(page);
  54. SetPageError(page);
  55. }
  56. unlock_page(page);
  57. } while (bvec >= bio->bi_io_vec);
  58. bio_put(bio);
  59. return 0;
  60. }
  61. static int mpage_end_io_write(struct bio *bio, unsigned int bytes_done, int err)
  62. {
  63. const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  64. struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
  65. if (bio->bi_size)
  66. return 1;
  67. do {
  68. struct page *page = bvec->bv_page;
  69. if (--bvec >= bio->bi_io_vec)
  70. prefetchw(&bvec->bv_page->flags);
  71. if (!uptodate){
  72. SetPageError(page);
  73. if (page->mapping)
  74. set_bit(AS_EIO, &page->mapping->flags);
  75. }
  76. end_page_writeback(page);
  77. } while (bvec >= bio->bi_io_vec);
  78. bio_put(bio);
  79. return 0;
  80. }
  81. static struct bio *mpage_bio_submit(int rw, struct bio *bio)
  82. {
  83. bio->bi_end_io = mpage_end_io_read;
  84. if (rw == WRITE)
  85. bio->bi_end_io = mpage_end_io_write;
  86. submit_bio(rw, bio);
  87. return NULL;
  88. }
  89. static struct bio *
  90. mpage_alloc(struct block_device *bdev,
  91. sector_t first_sector, int nr_vecs,
  92. gfp_t gfp_flags)
  93. {
  94. struct bio *bio;
  95. bio = bio_alloc(gfp_flags, nr_vecs);
  96. if (bio == NULL && (current->flags & PF_MEMALLOC)) {
  97. while (!bio && (nr_vecs /= 2))
  98. bio = bio_alloc(gfp_flags, nr_vecs);
  99. }
  100. if (bio) {
  101. bio->bi_bdev = bdev;
  102. bio->bi_sector = first_sector;
  103. }
  104. return bio;
  105. }
  106. /*
  107. * support function for mpage_readpages. The fs supplied get_block might
  108. * return an up to date buffer. This is used to map that buffer into
  109. * the page, which allows readpage to avoid triggering a duplicate call
  110. * to get_block.
  111. *
  112. * The idea is to avoid adding buffers to pages that don't already have
  113. * them. So when the buffer is up to date and the page size == block size,
  114. * this marks the page up to date instead of adding new buffers.
  115. */
  116. static void
  117. map_buffer_to_page(struct page *page, struct buffer_head *bh, int page_block)
  118. {
  119. struct inode *inode = page->mapping->host;
  120. struct buffer_head *page_bh, *head;
  121. int block = 0;
  122. if (!page_has_buffers(page)) {
  123. /*
  124. * don't make any buffers if there is only one buffer on
  125. * the page and the page just needs to be set up to date
  126. */
  127. if (inode->i_blkbits == PAGE_CACHE_SHIFT &&
  128. buffer_uptodate(bh)) {
  129. SetPageUptodate(page);
  130. return;
  131. }
  132. create_empty_buffers(page, 1 << inode->i_blkbits, 0);
  133. }
  134. head = page_buffers(page);
  135. page_bh = head;
  136. do {
  137. if (block == page_block) {
  138. page_bh->b_state = bh->b_state;
  139. page_bh->b_bdev = bh->b_bdev;
  140. page_bh->b_blocknr = bh->b_blocknr;
  141. break;
  142. }
  143. page_bh = page_bh->b_this_page;
  144. block++;
  145. } while (page_bh != head);
  146. }
  147. /*
  148. * This is the worker routine which does all the work of mapping the disk
  149. * blocks and constructs largest possible bios, submits them for IO if the
  150. * blocks are not contiguous on the disk.
  151. *
  152. * We pass a buffer_head back and forth and use its buffer_mapped() flag to
  153. * represent the validity of its disk mapping and to decide when to do the next
  154. * get_block() call.
  155. */
  156. static struct bio *
  157. do_mpage_readpage(struct bio *bio, struct page *page, unsigned nr_pages,
  158. sector_t *last_block_in_bio, struct buffer_head *map_bh,
  159. unsigned long *first_logical_block, get_block_t get_block)
  160. {
  161. struct inode *inode = page->mapping->host;
  162. const unsigned blkbits = inode->i_blkbits;
  163. const unsigned blocks_per_page = PAGE_CACHE_SIZE >> blkbits;
  164. const unsigned blocksize = 1 << blkbits;
  165. sector_t block_in_file;
  166. sector_t last_block;
  167. sector_t last_block_in_file;
  168. sector_t blocks[MAX_BUF_PER_PAGE];
  169. unsigned page_block;
  170. unsigned first_hole = blocks_per_page;
  171. struct block_device *bdev = NULL;
  172. int length;
  173. int fully_mapped = 1;
  174. unsigned nblocks;
  175. unsigned relative_block;
  176. if (page_has_buffers(page))
  177. goto confused;
  178. block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
  179. last_block = block_in_file + nr_pages * blocks_per_page;
  180. last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
  181. if (last_block > last_block_in_file)
  182. last_block = last_block_in_file;
  183. page_block = 0;
  184. /*
  185. * Map blocks using the result from the previous get_blocks call first.
  186. */
  187. nblocks = map_bh->b_size >> blkbits;
  188. if (buffer_mapped(map_bh) && block_in_file > *first_logical_block &&
  189. block_in_file < (*first_logical_block + nblocks)) {
  190. unsigned map_offset = block_in_file - *first_logical_block;
  191. unsigned last = nblocks - map_offset;
  192. for (relative_block = 0; ; relative_block++) {
  193. if (relative_block == last) {
  194. clear_buffer_mapped(map_bh);
  195. break;
  196. }
  197. if (page_block == blocks_per_page)
  198. break;
  199. blocks[page_block] = map_bh->b_blocknr + map_offset +
  200. relative_block;
  201. page_block++;
  202. block_in_file++;
  203. }
  204. bdev = map_bh->b_bdev;
  205. }
  206. /*
  207. * Then do more get_blocks calls until we are done with this page.
  208. */
  209. map_bh->b_page = page;
  210. while (page_block < blocks_per_page) {
  211. map_bh->b_state = 0;
  212. map_bh->b_size = 0;
  213. if (block_in_file < last_block) {
  214. map_bh->b_size = (last_block-block_in_file) << blkbits;
  215. if (get_block(inode, block_in_file, map_bh, 0))
  216. goto confused;
  217. *first_logical_block = block_in_file;
  218. }
  219. if (!buffer_mapped(map_bh)) {
  220. fully_mapped = 0;
  221. if (first_hole == blocks_per_page)
  222. first_hole = page_block;
  223. page_block++;
  224. block_in_file++;
  225. clear_buffer_mapped(map_bh);
  226. continue;
  227. }
  228. /* some filesystems will copy data into the page during
  229. * the get_block call, in which case we don't want to
  230. * read it again. map_buffer_to_page copies the data
  231. * we just collected from get_block into the page's buffers
  232. * so readpage doesn't have to repeat the get_block call
  233. */
  234. if (buffer_uptodate(map_bh)) {
  235. map_buffer_to_page(page, map_bh, page_block);
  236. goto confused;
  237. }
  238. if (first_hole != blocks_per_page)
  239. goto confused; /* hole -> non-hole */
  240. /* Contiguous blocks? */
  241. if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1)
  242. goto confused;
  243. nblocks = map_bh->b_size >> blkbits;
  244. for (relative_block = 0; ; relative_block++) {
  245. if (relative_block == nblocks) {
  246. clear_buffer_mapped(map_bh);
  247. break;
  248. } else if (page_block == blocks_per_page)
  249. break;
  250. blocks[page_block] = map_bh->b_blocknr+relative_block;
  251. page_block++;
  252. block_in_file++;
  253. }
  254. bdev = map_bh->b_bdev;
  255. }
  256. if (first_hole != blocks_per_page) {
  257. char *kaddr = kmap_atomic(page, KM_USER0);
  258. memset(kaddr + (first_hole << blkbits), 0,
  259. PAGE_CACHE_SIZE - (first_hole << blkbits));
  260. flush_dcache_page(page);
  261. kunmap_atomic(kaddr, KM_USER0);
  262. if (first_hole == 0) {
  263. SetPageUptodate(page);
  264. unlock_page(page);
  265. goto out;
  266. }
  267. } else if (fully_mapped) {
  268. SetPageMappedToDisk(page);
  269. }
  270. /*
  271. * This page will go to BIO. Do we need to send this BIO off first?
  272. */
  273. if (bio && (*last_block_in_bio != blocks[0] - 1))
  274. bio = mpage_bio_submit(READ, bio);
  275. alloc_new:
  276. if (bio == NULL) {
  277. bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
  278. min_t(int, nr_pages, bio_get_nr_vecs(bdev)),
  279. GFP_KERNEL);
  280. if (bio == NULL)
  281. goto confused;
  282. }
  283. length = first_hole << blkbits;
  284. if (bio_add_page(bio, page, length, 0) < length) {
  285. bio = mpage_bio_submit(READ, bio);
  286. goto alloc_new;
  287. }
  288. if (buffer_boundary(map_bh) || (first_hole != blocks_per_page))
  289. bio = mpage_bio_submit(READ, bio);
  290. else
  291. *last_block_in_bio = blocks[blocks_per_page - 1];
  292. out:
  293. return bio;
  294. confused:
  295. if (bio)
  296. bio = mpage_bio_submit(READ, bio);
  297. if (!PageUptodate(page))
  298. block_read_full_page(page, get_block);
  299. else
  300. unlock_page(page);
  301. goto out;
  302. }
  303. /**
  304. * mpage_readpages - populate an address space with some pages, and
  305. * start reads against them.
  306. *
  307. * @mapping: the address_space
  308. * @pages: The address of a list_head which contains the target pages. These
  309. * pages have their ->index populated and are otherwise uninitialised.
  310. *
  311. * The page at @pages->prev has the lowest file offset, and reads should be
  312. * issued in @pages->prev to @pages->next order.
  313. *
  314. * @nr_pages: The number of pages at *@pages
  315. * @get_block: The filesystem's block mapper function.
  316. *
  317. * This function walks the pages and the blocks within each page, building and
  318. * emitting large BIOs.
  319. *
  320. * If anything unusual happens, such as:
  321. *
  322. * - encountering a page which has buffers
  323. * - encountering a page which has a non-hole after a hole
  324. * - encountering a page with non-contiguous blocks
  325. *
  326. * then this code just gives up and calls the buffer_head-based read function.
  327. * It does handle a page which has holes at the end - that is a common case:
  328. * the end-of-file on blocksize < PAGE_CACHE_SIZE setups.
  329. *
  330. * BH_Boundary explanation:
  331. *
  332. * There is a problem. The mpage read code assembles several pages, gets all
  333. * their disk mappings, and then submits them all. That's fine, but obtaining
  334. * the disk mappings may require I/O. Reads of indirect blocks, for example.
  335. *
  336. * So an mpage read of the first 16 blocks of an ext2 file will cause I/O to be
  337. * submitted in the following order:
  338. * 12 0 1 2 3 4 5 6 7 8 9 10 11 13 14 15 16
  339. * because the indirect block has to be read to get the mappings of blocks
  340. * 13,14,15,16. Obviously, this impacts performance.
  341. *
  342. * So what we do it to allow the filesystem's get_block() function to set
  343. * BH_Boundary when it maps block 11. BH_Boundary says: mapping of the block
  344. * after this one will require I/O against a block which is probably close to
  345. * this one. So you should push what I/O you have currently accumulated.
  346. *
  347. * This all causes the disk requests to be issued in the correct order.
  348. */
  349. int
  350. mpage_readpages(struct address_space *mapping, struct list_head *pages,
  351. unsigned nr_pages, get_block_t get_block)
  352. {
  353. struct bio *bio = NULL;
  354. unsigned page_idx;
  355. sector_t last_block_in_bio = 0;
  356. struct pagevec lru_pvec;
  357. struct buffer_head map_bh;
  358. unsigned long first_logical_block = 0;
  359. clear_buffer_mapped(&map_bh);
  360. pagevec_init(&lru_pvec, 0);
  361. for (page_idx = 0; page_idx < nr_pages; page_idx++) {
  362. struct page *page = list_entry(pages->prev, struct page, lru);
  363. prefetchw(&page->flags);
  364. list_del(&page->lru);
  365. if (!add_to_page_cache(page, mapping,
  366. page->index, GFP_KERNEL)) {
  367. bio = do_mpage_readpage(bio, page,
  368. nr_pages - page_idx,
  369. &last_block_in_bio, &map_bh,
  370. &first_logical_block,
  371. get_block);
  372. if (!pagevec_add(&lru_pvec, page))
  373. __pagevec_lru_add(&lru_pvec);
  374. } else {
  375. page_cache_release(page);
  376. }
  377. }
  378. pagevec_lru_add(&lru_pvec);
  379. BUG_ON(!list_empty(pages));
  380. if (bio)
  381. mpage_bio_submit(READ, bio);
  382. return 0;
  383. }
  384. EXPORT_SYMBOL(mpage_readpages);
  385. /*
  386. * This isn't called much at all
  387. */
  388. int mpage_readpage(struct page *page, get_block_t get_block)
  389. {
  390. struct bio *bio = NULL;
  391. sector_t last_block_in_bio = 0;
  392. struct buffer_head map_bh;
  393. unsigned long first_logical_block = 0;
  394. clear_buffer_mapped(&map_bh);
  395. bio = do_mpage_readpage(bio, page, 1, &last_block_in_bio,
  396. &map_bh, &first_logical_block, get_block);
  397. if (bio)
  398. mpage_bio_submit(READ, bio);
  399. return 0;
  400. }
  401. EXPORT_SYMBOL(mpage_readpage);
  402. /*
  403. * Writing is not so simple.
  404. *
  405. * If the page has buffers then they will be used for obtaining the disk
  406. * mapping. We only support pages which are fully mapped-and-dirty, with a
  407. * special case for pages which are unmapped at the end: end-of-file.
  408. *
  409. * If the page has no buffers (preferred) then the page is mapped here.
  410. *
  411. * If all blocks are found to be contiguous then the page can go into the
  412. * BIO. Otherwise fall back to the mapping's writepage().
  413. *
  414. * FIXME: This code wants an estimate of how many pages are still to be
  415. * written, so it can intelligently allocate a suitably-sized BIO. For now,
  416. * just allocate full-size (16-page) BIOs.
  417. */
  418. static struct bio *
  419. __mpage_writepage(struct bio *bio, struct page *page, get_block_t get_block,
  420. sector_t *last_block_in_bio, int *ret, struct writeback_control *wbc,
  421. writepage_t writepage_fn)
  422. {
  423. struct address_space *mapping = page->mapping;
  424. struct inode *inode = page->mapping->host;
  425. const unsigned blkbits = inode->i_blkbits;
  426. unsigned long end_index;
  427. const unsigned blocks_per_page = PAGE_CACHE_SIZE >> blkbits;
  428. sector_t last_block;
  429. sector_t block_in_file;
  430. sector_t blocks[MAX_BUF_PER_PAGE];
  431. unsigned page_block;
  432. unsigned first_unmapped = blocks_per_page;
  433. struct block_device *bdev = NULL;
  434. int boundary = 0;
  435. sector_t boundary_block = 0;
  436. struct block_device *boundary_bdev = NULL;
  437. int length;
  438. struct buffer_head map_bh;
  439. loff_t i_size = i_size_read(inode);
  440. if (page_has_buffers(page)) {
  441. struct buffer_head *head = page_buffers(page);
  442. struct buffer_head *bh = head;
  443. /* If they're all mapped and dirty, do it */
  444. page_block = 0;
  445. do {
  446. BUG_ON(buffer_locked(bh));
  447. if (!buffer_mapped(bh)) {
  448. /*
  449. * unmapped dirty buffers are created by
  450. * __set_page_dirty_buffers -> mmapped data
  451. */
  452. if (buffer_dirty(bh))
  453. goto confused;
  454. if (first_unmapped == blocks_per_page)
  455. first_unmapped = page_block;
  456. continue;
  457. }
  458. if (first_unmapped != blocks_per_page)
  459. goto confused; /* hole -> non-hole */
  460. if (!buffer_dirty(bh) || !buffer_uptodate(bh))
  461. goto confused;
  462. if (page_block) {
  463. if (bh->b_blocknr != blocks[page_block-1] + 1)
  464. goto confused;
  465. }
  466. blocks[page_block++] = bh->b_blocknr;
  467. boundary = buffer_boundary(bh);
  468. if (boundary) {
  469. boundary_block = bh->b_blocknr;
  470. boundary_bdev = bh->b_bdev;
  471. }
  472. bdev = bh->b_bdev;
  473. } while ((bh = bh->b_this_page) != head);
  474. if (first_unmapped)
  475. goto page_is_mapped;
  476. /*
  477. * Page has buffers, but they are all unmapped. The page was
  478. * created by pagein or read over a hole which was handled by
  479. * block_read_full_page(). If this address_space is also
  480. * using mpage_readpages then this can rarely happen.
  481. */
  482. goto confused;
  483. }
  484. /*
  485. * The page has no buffers: map it to disk
  486. */
  487. BUG_ON(!PageUptodate(page));
  488. block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
  489. last_block = (i_size - 1) >> blkbits;
  490. map_bh.b_page = page;
  491. for (page_block = 0; page_block < blocks_per_page; ) {
  492. map_bh.b_state = 0;
  493. map_bh.b_size = 1 << blkbits;
  494. if (get_block(inode, block_in_file, &map_bh, 1))
  495. goto confused;
  496. if (buffer_new(&map_bh))
  497. unmap_underlying_metadata(map_bh.b_bdev,
  498. map_bh.b_blocknr);
  499. if (buffer_boundary(&map_bh)) {
  500. boundary_block = map_bh.b_blocknr;
  501. boundary_bdev = map_bh.b_bdev;
  502. }
  503. if (page_block) {
  504. if (map_bh.b_blocknr != blocks[page_block-1] + 1)
  505. goto confused;
  506. }
  507. blocks[page_block++] = map_bh.b_blocknr;
  508. boundary = buffer_boundary(&map_bh);
  509. bdev = map_bh.b_bdev;
  510. if (block_in_file == last_block)
  511. break;
  512. block_in_file++;
  513. }
  514. BUG_ON(page_block == 0);
  515. first_unmapped = page_block;
  516. page_is_mapped:
  517. end_index = i_size >> PAGE_CACHE_SHIFT;
  518. if (page->index >= end_index) {
  519. /*
  520. * The page straddles i_size. It must be zeroed out on each
  521. * and every writepage invokation because it may be mmapped.
  522. * "A file is mapped in multiples of the page size. For a file
  523. * that is not a multiple of the page size, the remaining memory
  524. * is zeroed when mapped, and writes to that region are not
  525. * written out to the file."
  526. */
  527. unsigned offset = i_size & (PAGE_CACHE_SIZE - 1);
  528. char *kaddr;
  529. if (page->index > end_index || !offset)
  530. goto confused;
  531. kaddr = kmap_atomic(page, KM_USER0);
  532. memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
  533. flush_dcache_page(page);
  534. kunmap_atomic(kaddr, KM_USER0);
  535. }
  536. /*
  537. * This page will go to BIO. Do we need to send this BIO off first?
  538. */
  539. if (bio && *last_block_in_bio != blocks[0] - 1)
  540. bio = mpage_bio_submit(WRITE, bio);
  541. alloc_new:
  542. if (bio == NULL) {
  543. bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
  544. bio_get_nr_vecs(bdev), GFP_NOFS|__GFP_HIGH);
  545. if (bio == NULL)
  546. goto confused;
  547. }
  548. /*
  549. * Must try to add the page before marking the buffer clean or
  550. * the confused fail path above (OOM) will be very confused when
  551. * it finds all bh marked clean (i.e. it will not write anything)
  552. */
  553. length = first_unmapped << blkbits;
  554. if (bio_add_page(bio, page, length, 0) < length) {
  555. bio = mpage_bio_submit(WRITE, bio);
  556. goto alloc_new;
  557. }
  558. /*
  559. * OK, we have our BIO, so we can now mark the buffers clean. Make
  560. * sure to only clean buffers which we know we'll be writing.
  561. */
  562. if (page_has_buffers(page)) {
  563. struct buffer_head *head = page_buffers(page);
  564. struct buffer_head *bh = head;
  565. unsigned buffer_counter = 0;
  566. do {
  567. if (buffer_counter++ == first_unmapped)
  568. break;
  569. clear_buffer_dirty(bh);
  570. bh = bh->b_this_page;
  571. } while (bh != head);
  572. /*
  573. * we cannot drop the bh if the page is not uptodate
  574. * or a concurrent readpage would fail to serialize with the bh
  575. * and it would read from disk before we reach the platter.
  576. */
  577. if (buffer_heads_over_limit && PageUptodate(page))
  578. try_to_free_buffers(page);
  579. }
  580. BUG_ON(PageWriteback(page));
  581. set_page_writeback(page);
  582. unlock_page(page);
  583. if (boundary || (first_unmapped != blocks_per_page)) {
  584. bio = mpage_bio_submit(WRITE, bio);
  585. if (boundary_block) {
  586. write_boundary_block(boundary_bdev,
  587. boundary_block, 1 << blkbits);
  588. }
  589. } else {
  590. *last_block_in_bio = blocks[blocks_per_page - 1];
  591. }
  592. goto out;
  593. confused:
  594. if (bio)
  595. bio = mpage_bio_submit(WRITE, bio);
  596. if (writepage_fn) {
  597. *ret = (*writepage_fn)(page, wbc);
  598. } else {
  599. *ret = -EAGAIN;
  600. goto out;
  601. }
  602. /*
  603. * The caller has a ref on the inode, so *mapping is stable
  604. */
  605. if (*ret) {
  606. if (*ret == -ENOSPC)
  607. set_bit(AS_ENOSPC, &mapping->flags);
  608. else
  609. set_bit(AS_EIO, &mapping->flags);
  610. }
  611. out:
  612. return bio;
  613. }
  614. /**
  615. * mpage_writepages - walk the list of dirty pages of the given
  616. * address space and writepage() all of them.
  617. *
  618. * @mapping: address space structure to write
  619. * @wbc: subtract the number of written pages from *@wbc->nr_to_write
  620. * @get_block: the filesystem's block mapper function.
  621. * If this is NULL then use a_ops->writepage. Otherwise, go
  622. * direct-to-BIO.
  623. *
  624. * This is a library function, which implements the writepages()
  625. * address_space_operation.
  626. *
  627. * If a page is already under I/O, generic_writepages() skips it, even
  628. * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
  629. * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
  630. * and msync() need to guarantee that all the data which was dirty at the time
  631. * the call was made get new I/O started against them. If wbc->sync_mode is
  632. * WB_SYNC_ALL then we were called for data integrity and we must wait for
  633. * existing IO to complete.
  634. *
  635. * If you fix this you should check generic_writepages() also!
  636. */
  637. int
  638. mpage_writepages(struct address_space *mapping,
  639. struct writeback_control *wbc, get_block_t get_block)
  640. {
  641. struct backing_dev_info *bdi = mapping->backing_dev_info;
  642. struct bio *bio = NULL;
  643. sector_t last_block_in_bio = 0;
  644. int ret = 0;
  645. int done = 0;
  646. int (*writepage)(struct page *page, struct writeback_control *wbc);
  647. struct pagevec pvec;
  648. int nr_pages;
  649. pgoff_t index;
  650. pgoff_t end; /* Inclusive */
  651. int scanned = 0;
  652. int range_whole = 0;
  653. if (wbc->nonblocking && bdi_write_congested(bdi)) {
  654. wbc->encountered_congestion = 1;
  655. return 0;
  656. }
  657. writepage = NULL;
  658. if (get_block == NULL)
  659. writepage = mapping->a_ops->writepage;
  660. pagevec_init(&pvec, 0);
  661. if (wbc->range_cyclic) {
  662. index = mapping->writeback_index; /* Start from prev offset */
  663. end = -1;
  664. } else {
  665. index = wbc->range_start >> PAGE_CACHE_SHIFT;
  666. end = wbc->range_end >> PAGE_CACHE_SHIFT;
  667. if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
  668. range_whole = 1;
  669. scanned = 1;
  670. }
  671. retry:
  672. while (!done && (index <= end) &&
  673. (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
  674. PAGECACHE_TAG_DIRTY,
  675. min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
  676. unsigned i;
  677. scanned = 1;
  678. for (i = 0; i < nr_pages; i++) {
  679. struct page *page = pvec.pages[i];
  680. /*
  681. * At this point we hold neither mapping->tree_lock nor
  682. * lock on the page itself: the page may be truncated or
  683. * invalidated (changing page->mapping to NULL), or even
  684. * swizzled back from swapper_space to tmpfs file
  685. * mapping
  686. */
  687. lock_page(page);
  688. if (unlikely(page->mapping != mapping)) {
  689. unlock_page(page);
  690. continue;
  691. }
  692. if (!wbc->range_cyclic && page->index > end) {
  693. done = 1;
  694. unlock_page(page);
  695. continue;
  696. }
  697. if (wbc->sync_mode != WB_SYNC_NONE)
  698. wait_on_page_writeback(page);
  699. if (PageWriteback(page) ||
  700. !clear_page_dirty_for_io(page)) {
  701. unlock_page(page);
  702. continue;
  703. }
  704. if (writepage) {
  705. ret = (*writepage)(page, wbc);
  706. if (ret) {
  707. if (ret == -ENOSPC)
  708. set_bit(AS_ENOSPC,
  709. &mapping->flags);
  710. else
  711. set_bit(AS_EIO,
  712. &mapping->flags);
  713. }
  714. } else {
  715. bio = __mpage_writepage(bio, page, get_block,
  716. &last_block_in_bio, &ret, wbc,
  717. page->mapping->a_ops->writepage);
  718. }
  719. if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE))
  720. unlock_page(page);
  721. if (ret || (--(wbc->nr_to_write) <= 0))
  722. done = 1;
  723. if (wbc->nonblocking && bdi_write_congested(bdi)) {
  724. wbc->encountered_congestion = 1;
  725. done = 1;
  726. }
  727. }
  728. pagevec_release(&pvec);
  729. cond_resched();
  730. }
  731. if (!scanned && !done) {
  732. /*
  733. * We hit the last page and there is more work to be done: wrap
  734. * back to the start of the file
  735. */
  736. scanned = 1;
  737. index = 0;
  738. goto retry;
  739. }
  740. if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
  741. mapping->writeback_index = index;
  742. if (bio)
  743. mpage_bio_submit(WRITE, bio);
  744. return ret;
  745. }
  746. EXPORT_SYMBOL(mpage_writepages);
  747. int mpage_writepage(struct page *page, get_block_t get_block,
  748. struct writeback_control *wbc)
  749. {
  750. int ret = 0;
  751. struct bio *bio;
  752. sector_t last_block_in_bio = 0;
  753. bio = __mpage_writepage(NULL, page, get_block,
  754. &last_block_in_bio, &ret, wbc, NULL);
  755. if (bio)
  756. mpage_bio_submit(WRITE, bio);
  757. return ret;
  758. }
  759. EXPORT_SYMBOL(mpage_writepage);