xfs_aops.c 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482
  1. /*
  2. * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  3. * All Rights Reserved.
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of the GNU General Public License as
  7. * published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it would be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write the Free Software Foundation,
  16. * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  17. */
  18. #include "xfs.h"
  19. #include "xfs_bit.h"
  20. #include "xfs_log.h"
  21. #include "xfs_inum.h"
  22. #include "xfs_sb.h"
  23. #include "xfs_ag.h"
  24. #include "xfs_dir2.h"
  25. #include "xfs_trans.h"
  26. #include "xfs_dmapi.h"
  27. #include "xfs_mount.h"
  28. #include "xfs_bmap_btree.h"
  29. #include "xfs_alloc_btree.h"
  30. #include "xfs_ialloc_btree.h"
  31. #include "xfs_dir2_sf.h"
  32. #include "xfs_attr_sf.h"
  33. #include "xfs_dinode.h"
  34. #include "xfs_inode.h"
  35. #include "xfs_alloc.h"
  36. #include "xfs_btree.h"
  37. #include "xfs_error.h"
  38. #include "xfs_rw.h"
  39. #include "xfs_iomap.h"
  40. #include <linux/mpage.h>
  41. #include <linux/pagevec.h>
  42. #include <linux/writeback.h>
  43. STATIC void
  44. xfs_count_page_state(
  45. struct page *page,
  46. int *delalloc,
  47. int *unmapped,
  48. int *unwritten)
  49. {
  50. struct buffer_head *bh, *head;
  51. *delalloc = *unmapped = *unwritten = 0;
  52. bh = head = page_buffers(page);
  53. do {
  54. if (buffer_uptodate(bh) && !buffer_mapped(bh))
  55. (*unmapped) = 1;
  56. else if (buffer_unwritten(bh))
  57. (*unwritten) = 1;
  58. else if (buffer_delay(bh))
  59. (*delalloc) = 1;
  60. } while ((bh = bh->b_this_page) != head);
  61. }
  62. #if defined(XFS_RW_TRACE)
  63. void
  64. xfs_page_trace(
  65. int tag,
  66. struct inode *inode,
  67. struct page *page,
  68. unsigned long pgoff)
  69. {
  70. xfs_inode_t *ip;
  71. bhv_vnode_t *vp = vn_from_inode(inode);
  72. loff_t isize = i_size_read(inode);
  73. loff_t offset = page_offset(page);
  74. int delalloc = -1, unmapped = -1, unwritten = -1;
  75. if (page_has_buffers(page))
  76. xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
  77. ip = xfs_vtoi(vp);
  78. if (!ip->i_rwtrace)
  79. return;
  80. ktrace_enter(ip->i_rwtrace,
  81. (void *)((unsigned long)tag),
  82. (void *)ip,
  83. (void *)inode,
  84. (void *)page,
  85. (void *)pgoff,
  86. (void *)((unsigned long)((ip->i_d.di_size >> 32) & 0xffffffff)),
  87. (void *)((unsigned long)(ip->i_d.di_size & 0xffffffff)),
  88. (void *)((unsigned long)((isize >> 32) & 0xffffffff)),
  89. (void *)((unsigned long)(isize & 0xffffffff)),
  90. (void *)((unsigned long)((offset >> 32) & 0xffffffff)),
  91. (void *)((unsigned long)(offset & 0xffffffff)),
  92. (void *)((unsigned long)delalloc),
  93. (void *)((unsigned long)unmapped),
  94. (void *)((unsigned long)unwritten),
  95. (void *)((unsigned long)current_pid()),
  96. (void *)NULL);
  97. }
  98. #else
  99. #define xfs_page_trace(tag, inode, page, pgoff)
  100. #endif
  101. /*
  102. * Schedule IO completion handling on a xfsdatad if this was
  103. * the final hold on this ioend.
  104. */
  105. STATIC void
  106. xfs_finish_ioend(
  107. xfs_ioend_t *ioend)
  108. {
  109. if (atomic_dec_and_test(&ioend->io_remaining))
  110. queue_work(xfsdatad_workqueue, &ioend->io_work);
  111. }
  112. /*
  113. * We're now finished for good with this ioend structure.
  114. * Update the page state via the associated buffer_heads,
  115. * release holds on the inode and bio, and finally free
  116. * up memory. Do not use the ioend after this.
  117. */
  118. STATIC void
  119. xfs_destroy_ioend(
  120. xfs_ioend_t *ioend)
  121. {
  122. struct buffer_head *bh, *next;
  123. for (bh = ioend->io_buffer_head; bh; bh = next) {
  124. next = bh->b_private;
  125. bh->b_end_io(bh, !ioend->io_error);
  126. }
  127. if (unlikely(ioend->io_error))
  128. vn_ioerror(ioend->io_vnode, ioend->io_error, __FILE__,__LINE__);
  129. vn_iowake(ioend->io_vnode);
  130. mempool_free(ioend, xfs_ioend_pool);
  131. }
  132. /*
  133. * Buffered IO write completion for delayed allocate extents.
  134. * TODO: Update ondisk isize now that we know the file data
  135. * has been flushed (i.e. the notorious "NULL file" problem).
  136. */
  137. STATIC void
  138. xfs_end_bio_delalloc(
  139. struct work_struct *work)
  140. {
  141. xfs_ioend_t *ioend =
  142. container_of(work, xfs_ioend_t, io_work);
  143. xfs_destroy_ioend(ioend);
  144. }
  145. /*
  146. * Buffered IO write completion for regular, written extents.
  147. */
  148. STATIC void
  149. xfs_end_bio_written(
  150. struct work_struct *work)
  151. {
  152. xfs_ioend_t *ioend =
  153. container_of(work, xfs_ioend_t, io_work);
  154. xfs_destroy_ioend(ioend);
  155. }
  156. /*
  157. * IO write completion for unwritten extents.
  158. *
  159. * Issue transactions to convert a buffer range from unwritten
  160. * to written extents.
  161. */
  162. STATIC void
  163. xfs_end_bio_unwritten(
  164. struct work_struct *work)
  165. {
  166. xfs_ioend_t *ioend =
  167. container_of(work, xfs_ioend_t, io_work);
  168. bhv_vnode_t *vp = ioend->io_vnode;
  169. xfs_off_t offset = ioend->io_offset;
  170. size_t size = ioend->io_size;
  171. if (likely(!ioend->io_error))
  172. bhv_vop_bmap(vp, offset, size, BMAPI_UNWRITTEN, NULL, NULL);
  173. xfs_destroy_ioend(ioend);
  174. }
  175. /*
  176. * Allocate and initialise an IO completion structure.
  177. * We need to track unwritten extent write completion here initially.
  178. * We'll need to extend this for updating the ondisk inode size later
  179. * (vs. incore size).
  180. */
  181. STATIC xfs_ioend_t *
  182. xfs_alloc_ioend(
  183. struct inode *inode,
  184. unsigned int type)
  185. {
  186. xfs_ioend_t *ioend;
  187. ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS);
  188. /*
  189. * Set the count to 1 initially, which will prevent an I/O
  190. * completion callback from happening before we have started
  191. * all the I/O from calling the completion routine too early.
  192. */
  193. atomic_set(&ioend->io_remaining, 1);
  194. ioend->io_error = 0;
  195. ioend->io_list = NULL;
  196. ioend->io_type = type;
  197. ioend->io_vnode = vn_from_inode(inode);
  198. ioend->io_buffer_head = NULL;
  199. ioend->io_buffer_tail = NULL;
  200. atomic_inc(&ioend->io_vnode->v_iocount);
  201. ioend->io_offset = 0;
  202. ioend->io_size = 0;
  203. if (type == IOMAP_UNWRITTEN)
  204. INIT_WORK(&ioend->io_work, xfs_end_bio_unwritten);
  205. else if (type == IOMAP_DELAY)
  206. INIT_WORK(&ioend->io_work, xfs_end_bio_delalloc);
  207. else
  208. INIT_WORK(&ioend->io_work, xfs_end_bio_written);
  209. return ioend;
  210. }
  211. STATIC int
  212. xfs_map_blocks(
  213. struct inode *inode,
  214. loff_t offset,
  215. ssize_t count,
  216. xfs_iomap_t *mapp,
  217. int flags)
  218. {
  219. bhv_vnode_t *vp = vn_from_inode(inode);
  220. int error, nmaps = 1;
  221. error = bhv_vop_bmap(vp, offset, count, flags, mapp, &nmaps);
  222. if (!error && (flags & (BMAPI_WRITE|BMAPI_ALLOCATE)))
  223. VMODIFY(vp);
  224. return -error;
  225. }
  226. STATIC_INLINE int
  227. xfs_iomap_valid(
  228. xfs_iomap_t *iomapp,
  229. loff_t offset)
  230. {
  231. return offset >= iomapp->iomap_offset &&
  232. offset < iomapp->iomap_offset + iomapp->iomap_bsize;
  233. }
  234. /*
  235. * BIO completion handler for buffered IO.
  236. */
  237. STATIC int
  238. xfs_end_bio(
  239. struct bio *bio,
  240. unsigned int bytes_done,
  241. int error)
  242. {
  243. xfs_ioend_t *ioend = bio->bi_private;
  244. if (bio->bi_size)
  245. return 1;
  246. ASSERT(atomic_read(&bio->bi_cnt) >= 1);
  247. ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error;
  248. /* Toss bio and pass work off to an xfsdatad thread */
  249. bio->bi_private = NULL;
  250. bio->bi_end_io = NULL;
  251. bio_put(bio);
  252. xfs_finish_ioend(ioend);
  253. return 0;
  254. }
  255. STATIC void
  256. xfs_submit_ioend_bio(
  257. xfs_ioend_t *ioend,
  258. struct bio *bio)
  259. {
  260. atomic_inc(&ioend->io_remaining);
  261. bio->bi_private = ioend;
  262. bio->bi_end_io = xfs_end_bio;
  263. submit_bio(WRITE, bio);
  264. ASSERT(!bio_flagged(bio, BIO_EOPNOTSUPP));
  265. bio_put(bio);
  266. }
  267. STATIC struct bio *
  268. xfs_alloc_ioend_bio(
  269. struct buffer_head *bh)
  270. {
  271. struct bio *bio;
  272. int nvecs = bio_get_nr_vecs(bh->b_bdev);
  273. do {
  274. bio = bio_alloc(GFP_NOIO, nvecs);
  275. nvecs >>= 1;
  276. } while (!bio);
  277. ASSERT(bio->bi_private == NULL);
  278. bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
  279. bio->bi_bdev = bh->b_bdev;
  280. bio_get(bio);
  281. return bio;
  282. }
  283. STATIC void
  284. xfs_start_buffer_writeback(
  285. struct buffer_head *bh)
  286. {
  287. ASSERT(buffer_mapped(bh));
  288. ASSERT(buffer_locked(bh));
  289. ASSERT(!buffer_delay(bh));
  290. ASSERT(!buffer_unwritten(bh));
  291. mark_buffer_async_write(bh);
  292. set_buffer_uptodate(bh);
  293. clear_buffer_dirty(bh);
  294. }
  295. STATIC void
  296. xfs_start_page_writeback(
  297. struct page *page,
  298. struct writeback_control *wbc,
  299. int clear_dirty,
  300. int buffers)
  301. {
  302. ASSERT(PageLocked(page));
  303. ASSERT(!PageWriteback(page));
  304. if (clear_dirty)
  305. clear_page_dirty_for_io(page);
  306. set_page_writeback(page);
  307. unlock_page(page);
  308. if (!buffers) {
  309. end_page_writeback(page);
  310. wbc->pages_skipped++; /* We didn't write this page */
  311. }
  312. }
  313. static inline int bio_add_buffer(struct bio *bio, struct buffer_head *bh)
  314. {
  315. return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
  316. }
  317. /*
  318. * Submit all of the bios for all of the ioends we have saved up, covering the
  319. * initial writepage page and also any probed pages.
  320. *
  321. * Because we may have multiple ioends spanning a page, we need to start
  322. * writeback on all the buffers before we submit them for I/O. If we mark the
  323. * buffers as we got, then we can end up with a page that only has buffers
  324. * marked async write and I/O complete on can occur before we mark the other
  325. * buffers async write.
  326. *
  327. * The end result of this is that we trip a bug in end_page_writeback() because
  328. * we call it twice for the one page as the code in end_buffer_async_write()
  329. * assumes that all buffers on the page are started at the same time.
  330. *
  331. * The fix is two passes across the ioend list - one to start writeback on the
  332. * buffer_heads, and then submit them for I/O on the second pass.
  333. */
  334. STATIC void
  335. xfs_submit_ioend(
  336. xfs_ioend_t *ioend)
  337. {
  338. xfs_ioend_t *head = ioend;
  339. xfs_ioend_t *next;
  340. struct buffer_head *bh;
  341. struct bio *bio;
  342. sector_t lastblock = 0;
  343. /* Pass 1 - start writeback */
  344. do {
  345. next = ioend->io_list;
  346. for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
  347. xfs_start_buffer_writeback(bh);
  348. }
  349. } while ((ioend = next) != NULL);
  350. /* Pass 2 - submit I/O */
  351. ioend = head;
  352. do {
  353. next = ioend->io_list;
  354. bio = NULL;
  355. for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
  356. if (!bio) {
  357. retry:
  358. bio = xfs_alloc_ioend_bio(bh);
  359. } else if (bh->b_blocknr != lastblock + 1) {
  360. xfs_submit_ioend_bio(ioend, bio);
  361. goto retry;
  362. }
  363. if (bio_add_buffer(bio, bh) != bh->b_size) {
  364. xfs_submit_ioend_bio(ioend, bio);
  365. goto retry;
  366. }
  367. lastblock = bh->b_blocknr;
  368. }
  369. if (bio)
  370. xfs_submit_ioend_bio(ioend, bio);
  371. xfs_finish_ioend(ioend);
  372. } while ((ioend = next) != NULL);
  373. }
  374. /*
  375. * Cancel submission of all buffer_heads so far in this endio.
  376. * Toss the endio too. Only ever called for the initial page
  377. * in a writepage request, so only ever one page.
  378. */
  379. STATIC void
  380. xfs_cancel_ioend(
  381. xfs_ioend_t *ioend)
  382. {
  383. xfs_ioend_t *next;
  384. struct buffer_head *bh, *next_bh;
  385. do {
  386. next = ioend->io_list;
  387. bh = ioend->io_buffer_head;
  388. do {
  389. next_bh = bh->b_private;
  390. clear_buffer_async_write(bh);
  391. unlock_buffer(bh);
  392. } while ((bh = next_bh) != NULL);
  393. vn_iowake(ioend->io_vnode);
  394. mempool_free(ioend, xfs_ioend_pool);
  395. } while ((ioend = next) != NULL);
  396. }
  397. /*
  398. * Test to see if we've been building up a completion structure for
  399. * earlier buffers -- if so, we try to append to this ioend if we
  400. * can, otherwise we finish off any current ioend and start another.
  401. * Return true if we've finished the given ioend.
  402. */
  403. STATIC void
  404. xfs_add_to_ioend(
  405. struct inode *inode,
  406. struct buffer_head *bh,
  407. xfs_off_t offset,
  408. unsigned int type,
  409. xfs_ioend_t **result,
  410. int need_ioend)
  411. {
  412. xfs_ioend_t *ioend = *result;
  413. if (!ioend || need_ioend || type != ioend->io_type) {
  414. xfs_ioend_t *previous = *result;
  415. ioend = xfs_alloc_ioend(inode, type);
  416. ioend->io_offset = offset;
  417. ioend->io_buffer_head = bh;
  418. ioend->io_buffer_tail = bh;
  419. if (previous)
  420. previous->io_list = ioend;
  421. *result = ioend;
  422. } else {
  423. ioend->io_buffer_tail->b_private = bh;
  424. ioend->io_buffer_tail = bh;
  425. }
  426. bh->b_private = NULL;
  427. ioend->io_size += bh->b_size;
  428. }
  429. STATIC void
  430. xfs_map_buffer(
  431. struct buffer_head *bh,
  432. xfs_iomap_t *mp,
  433. xfs_off_t offset,
  434. uint block_bits)
  435. {
  436. sector_t bn;
  437. ASSERT(mp->iomap_bn != IOMAP_DADDR_NULL);
  438. bn = (mp->iomap_bn >> (block_bits - BBSHIFT)) +
  439. ((offset - mp->iomap_offset) >> block_bits);
  440. ASSERT(bn || (mp->iomap_flags & IOMAP_REALTIME));
  441. bh->b_blocknr = bn;
  442. set_buffer_mapped(bh);
  443. }
  444. STATIC void
  445. xfs_map_at_offset(
  446. struct buffer_head *bh,
  447. loff_t offset,
  448. int block_bits,
  449. xfs_iomap_t *iomapp)
  450. {
  451. ASSERT(!(iomapp->iomap_flags & IOMAP_HOLE));
  452. ASSERT(!(iomapp->iomap_flags & IOMAP_DELAY));
  453. lock_buffer(bh);
  454. xfs_map_buffer(bh, iomapp, offset, block_bits);
  455. bh->b_bdev = iomapp->iomap_target->bt_bdev;
  456. set_buffer_mapped(bh);
  457. clear_buffer_delay(bh);
  458. clear_buffer_unwritten(bh);
  459. }
  460. /*
  461. * Look for a page at index that is suitable for clustering.
  462. */
  463. STATIC unsigned int
  464. xfs_probe_page(
  465. struct page *page,
  466. unsigned int pg_offset,
  467. int mapped)
  468. {
  469. int ret = 0;
  470. if (PageWriteback(page))
  471. return 0;
  472. if (page->mapping && PageDirty(page)) {
  473. if (page_has_buffers(page)) {
  474. struct buffer_head *bh, *head;
  475. bh = head = page_buffers(page);
  476. do {
  477. if (!buffer_uptodate(bh))
  478. break;
  479. if (mapped != buffer_mapped(bh))
  480. break;
  481. ret += bh->b_size;
  482. if (ret >= pg_offset)
  483. break;
  484. } while ((bh = bh->b_this_page) != head);
  485. } else
  486. ret = mapped ? 0 : PAGE_CACHE_SIZE;
  487. }
  488. return ret;
  489. }
  490. STATIC size_t
  491. xfs_probe_cluster(
  492. struct inode *inode,
  493. struct page *startpage,
  494. struct buffer_head *bh,
  495. struct buffer_head *head,
  496. int mapped)
  497. {
  498. struct pagevec pvec;
  499. pgoff_t tindex, tlast, tloff;
  500. size_t total = 0;
  501. int done = 0, i;
  502. /* First sum forwards in this page */
  503. do {
  504. if (!buffer_uptodate(bh) || (mapped != buffer_mapped(bh)))
  505. return total;
  506. total += bh->b_size;
  507. } while ((bh = bh->b_this_page) != head);
  508. /* if we reached the end of the page, sum forwards in following pages */
  509. tlast = i_size_read(inode) >> PAGE_CACHE_SHIFT;
  510. tindex = startpage->index + 1;
  511. /* Prune this back to avoid pathological behavior */
  512. tloff = min(tlast, startpage->index + 64);
  513. pagevec_init(&pvec, 0);
  514. while (!done && tindex <= tloff) {
  515. unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
  516. if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
  517. break;
  518. for (i = 0; i < pagevec_count(&pvec); i++) {
  519. struct page *page = pvec.pages[i];
  520. size_t pg_offset, len = 0;
  521. if (tindex == tlast) {
  522. pg_offset =
  523. i_size_read(inode) & (PAGE_CACHE_SIZE - 1);
  524. if (!pg_offset) {
  525. done = 1;
  526. break;
  527. }
  528. } else
  529. pg_offset = PAGE_CACHE_SIZE;
  530. if (page->index == tindex && !TestSetPageLocked(page)) {
  531. len = xfs_probe_page(page, pg_offset, mapped);
  532. unlock_page(page);
  533. }
  534. if (!len) {
  535. done = 1;
  536. break;
  537. }
  538. total += len;
  539. tindex++;
  540. }
  541. pagevec_release(&pvec);
  542. cond_resched();
  543. }
  544. return total;
  545. }
  546. /*
  547. * Test if a given page is suitable for writing as part of an unwritten
  548. * or delayed allocate extent.
  549. */
  550. STATIC int
  551. xfs_is_delayed_page(
  552. struct page *page,
  553. unsigned int type)
  554. {
  555. if (PageWriteback(page))
  556. return 0;
  557. if (page->mapping && page_has_buffers(page)) {
  558. struct buffer_head *bh, *head;
  559. int acceptable = 0;
  560. bh = head = page_buffers(page);
  561. do {
  562. if (buffer_unwritten(bh))
  563. acceptable = (type == IOMAP_UNWRITTEN);
  564. else if (buffer_delay(bh))
  565. acceptable = (type == IOMAP_DELAY);
  566. else if (buffer_dirty(bh) && buffer_mapped(bh))
  567. acceptable = (type == 0);
  568. else
  569. break;
  570. } while ((bh = bh->b_this_page) != head);
  571. if (acceptable)
  572. return 1;
  573. }
  574. return 0;
  575. }
  576. /*
  577. * Allocate & map buffers for page given the extent map. Write it out.
  578. * except for the original page of a writepage, this is called on
  579. * delalloc/unwritten pages only, for the original page it is possible
  580. * that the page has no mapping at all.
  581. */
  582. STATIC int
  583. xfs_convert_page(
  584. struct inode *inode,
  585. struct page *page,
  586. loff_t tindex,
  587. xfs_iomap_t *mp,
  588. xfs_ioend_t **ioendp,
  589. struct writeback_control *wbc,
  590. int startio,
  591. int all_bh)
  592. {
  593. struct buffer_head *bh, *head;
  594. xfs_off_t end_offset;
  595. unsigned long p_offset;
  596. unsigned int type;
  597. int bbits = inode->i_blkbits;
  598. int len, page_dirty;
  599. int count = 0, done = 0, uptodate = 1;
  600. xfs_off_t offset = page_offset(page);
  601. if (page->index != tindex)
  602. goto fail;
  603. if (TestSetPageLocked(page))
  604. goto fail;
  605. if (PageWriteback(page))
  606. goto fail_unlock_page;
  607. if (page->mapping != inode->i_mapping)
  608. goto fail_unlock_page;
  609. if (!xfs_is_delayed_page(page, (*ioendp)->io_type))
  610. goto fail_unlock_page;
  611. /*
  612. * page_dirty is initially a count of buffers on the page before
  613. * EOF and is decremented as we move each into a cleanable state.
  614. *
  615. * Derivation:
  616. *
  617. * End offset is the highest offset that this page should represent.
  618. * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
  619. * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
  620. * hence give us the correct page_dirty count. On any other page,
  621. * it will be zero and in that case we need page_dirty to be the
  622. * count of buffers on the page.
  623. */
  624. end_offset = min_t(unsigned long long,
  625. (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
  626. i_size_read(inode));
  627. len = 1 << inode->i_blkbits;
  628. p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
  629. PAGE_CACHE_SIZE);
  630. p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
  631. page_dirty = p_offset / len;
  632. bh = head = page_buffers(page);
  633. do {
  634. if (offset >= end_offset)
  635. break;
  636. if (!buffer_uptodate(bh))
  637. uptodate = 0;
  638. if (!(PageUptodate(page) || buffer_uptodate(bh))) {
  639. done = 1;
  640. continue;
  641. }
  642. if (buffer_unwritten(bh) || buffer_delay(bh)) {
  643. if (buffer_unwritten(bh))
  644. type = IOMAP_UNWRITTEN;
  645. else
  646. type = IOMAP_DELAY;
  647. if (!xfs_iomap_valid(mp, offset)) {
  648. done = 1;
  649. continue;
  650. }
  651. ASSERT(!(mp->iomap_flags & IOMAP_HOLE));
  652. ASSERT(!(mp->iomap_flags & IOMAP_DELAY));
  653. xfs_map_at_offset(bh, offset, bbits, mp);
  654. if (startio) {
  655. xfs_add_to_ioend(inode, bh, offset,
  656. type, ioendp, done);
  657. } else {
  658. set_buffer_dirty(bh);
  659. unlock_buffer(bh);
  660. mark_buffer_dirty(bh);
  661. }
  662. page_dirty--;
  663. count++;
  664. } else {
  665. type = 0;
  666. if (buffer_mapped(bh) && all_bh && startio) {
  667. lock_buffer(bh);
  668. xfs_add_to_ioend(inode, bh, offset,
  669. type, ioendp, done);
  670. count++;
  671. page_dirty--;
  672. } else {
  673. done = 1;
  674. }
  675. }
  676. } while (offset += len, (bh = bh->b_this_page) != head);
  677. if (uptodate && bh == head)
  678. SetPageUptodate(page);
  679. if (startio) {
  680. if (count) {
  681. struct backing_dev_info *bdi;
  682. bdi = inode->i_mapping->backing_dev_info;
  683. wbc->nr_to_write--;
  684. if (bdi_write_congested(bdi)) {
  685. wbc->encountered_congestion = 1;
  686. done = 1;
  687. } else if (wbc->nr_to_write <= 0) {
  688. done = 1;
  689. }
  690. }
  691. xfs_start_page_writeback(page, wbc, !page_dirty, count);
  692. }
  693. return done;
  694. fail_unlock_page:
  695. unlock_page(page);
  696. fail:
  697. return 1;
  698. }
  699. /*
  700. * Convert & write out a cluster of pages in the same extent as defined
  701. * by mp and following the start page.
  702. */
  703. STATIC void
  704. xfs_cluster_write(
  705. struct inode *inode,
  706. pgoff_t tindex,
  707. xfs_iomap_t *iomapp,
  708. xfs_ioend_t **ioendp,
  709. struct writeback_control *wbc,
  710. int startio,
  711. int all_bh,
  712. pgoff_t tlast)
  713. {
  714. struct pagevec pvec;
  715. int done = 0, i;
  716. pagevec_init(&pvec, 0);
  717. while (!done && tindex <= tlast) {
  718. unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
  719. if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
  720. break;
  721. for (i = 0; i < pagevec_count(&pvec); i++) {
  722. done = xfs_convert_page(inode, pvec.pages[i], tindex++,
  723. iomapp, ioendp, wbc, startio, all_bh);
  724. if (done)
  725. break;
  726. }
  727. pagevec_release(&pvec);
  728. cond_resched();
  729. }
  730. }
  731. /*
  732. * Calling this without startio set means we are being asked to make a dirty
  733. * page ready for freeing it's buffers. When called with startio set then
  734. * we are coming from writepage.
  735. *
  736. * When called with startio set it is important that we write the WHOLE
  737. * page if possible.
  738. * The bh->b_state's cannot know if any of the blocks or which block for
  739. * that matter are dirty due to mmap writes, and therefore bh uptodate is
  740. * only valid if the page itself isn't completely uptodate. Some layers
  741. * may clear the page dirty flag prior to calling write page, under the
  742. * assumption the entire page will be written out; by not writing out the
  743. * whole page the page can be reused before all valid dirty data is
  744. * written out. Note: in the case of a page that has been dirty'd by
  745. * mapwrite and but partially setup by block_prepare_write the
  746. * bh->b_states's will not agree and only ones setup by BPW/BCW will have
  747. * valid state, thus the whole page must be written out thing.
  748. */
  749. STATIC int
  750. xfs_page_state_convert(
  751. struct inode *inode,
  752. struct page *page,
  753. struct writeback_control *wbc,
  754. int startio,
  755. int unmapped) /* also implies page uptodate */
  756. {
  757. struct buffer_head *bh, *head;
  758. xfs_iomap_t iomap;
  759. xfs_ioend_t *ioend = NULL, *iohead = NULL;
  760. loff_t offset;
  761. unsigned long p_offset = 0;
  762. unsigned int type;
  763. __uint64_t end_offset;
  764. pgoff_t end_index, last_index, tlast;
  765. ssize_t size, len;
  766. int flags, err, iomap_valid = 0, uptodate = 1;
  767. int page_dirty, count = 0;
  768. int trylock = 0;
  769. int all_bh = unmapped;
  770. if (startio) {
  771. if (wbc->sync_mode == WB_SYNC_NONE && wbc->nonblocking)
  772. trylock |= BMAPI_TRYLOCK;
  773. }
  774. /* Is this page beyond the end of the file? */
  775. offset = i_size_read(inode);
  776. end_index = offset >> PAGE_CACHE_SHIFT;
  777. last_index = (offset - 1) >> PAGE_CACHE_SHIFT;
  778. if (page->index >= end_index) {
  779. if ((page->index >= end_index + 1) ||
  780. !(i_size_read(inode) & (PAGE_CACHE_SIZE - 1))) {
  781. if (startio)
  782. unlock_page(page);
  783. return 0;
  784. }
  785. }
  786. /*
  787. * page_dirty is initially a count of buffers on the page before
  788. * EOF and is decremented as we move each into a cleanable state.
  789. *
  790. * Derivation:
  791. *
  792. * End offset is the highest offset that this page should represent.
  793. * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
  794. * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
  795. * hence give us the correct page_dirty count. On any other page,
  796. * it will be zero and in that case we need page_dirty to be the
  797. * count of buffers on the page.
  798. */
  799. end_offset = min_t(unsigned long long,
  800. (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT, offset);
  801. len = 1 << inode->i_blkbits;
  802. p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
  803. PAGE_CACHE_SIZE);
  804. p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
  805. page_dirty = p_offset / len;
  806. bh = head = page_buffers(page);
  807. offset = page_offset(page);
  808. flags = -1;
  809. type = 0;
  810. /* TODO: cleanup count and page_dirty */
  811. do {
  812. if (offset >= end_offset)
  813. break;
  814. if (!buffer_uptodate(bh))
  815. uptodate = 0;
  816. if (!(PageUptodate(page) || buffer_uptodate(bh)) && !startio) {
  817. /*
  818. * the iomap is actually still valid, but the ioend
  819. * isn't. shouldn't happen too often.
  820. */
  821. iomap_valid = 0;
  822. continue;
  823. }
  824. if (iomap_valid)
  825. iomap_valid = xfs_iomap_valid(&iomap, offset);
  826. /*
  827. * First case, map an unwritten extent and prepare for
  828. * extent state conversion transaction on completion.
  829. *
  830. * Second case, allocate space for a delalloc buffer.
  831. * We can return EAGAIN here in the release page case.
  832. *
  833. * Third case, an unmapped buffer was found, and we are
  834. * in a path where we need to write the whole page out.
  835. */
  836. if (buffer_unwritten(bh) || buffer_delay(bh) ||
  837. ((buffer_uptodate(bh) || PageUptodate(page)) &&
  838. !buffer_mapped(bh) && (unmapped || startio))) {
  839. /*
  840. * Make sure we don't use a read-only iomap
  841. */
  842. if (flags == BMAPI_READ)
  843. iomap_valid = 0;
  844. if (buffer_unwritten(bh)) {
  845. type = IOMAP_UNWRITTEN;
  846. flags = BMAPI_WRITE | BMAPI_IGNSTATE;
  847. } else if (buffer_delay(bh)) {
  848. type = IOMAP_DELAY;
  849. flags = BMAPI_ALLOCATE | trylock;
  850. } else {
  851. type = IOMAP_NEW;
  852. flags = BMAPI_WRITE | BMAPI_MMAP;
  853. }
  854. if (!iomap_valid) {
  855. if (type == IOMAP_NEW) {
  856. size = xfs_probe_cluster(inode,
  857. page, bh, head, 0);
  858. } else {
  859. size = len;
  860. }
  861. err = xfs_map_blocks(inode, offset, size,
  862. &iomap, flags);
  863. if (err)
  864. goto error;
  865. iomap_valid = xfs_iomap_valid(&iomap, offset);
  866. }
  867. if (iomap_valid) {
  868. xfs_map_at_offset(bh, offset,
  869. inode->i_blkbits, &iomap);
  870. if (startio) {
  871. xfs_add_to_ioend(inode, bh, offset,
  872. type, &ioend,
  873. !iomap_valid);
  874. } else {
  875. set_buffer_dirty(bh);
  876. unlock_buffer(bh);
  877. mark_buffer_dirty(bh);
  878. }
  879. page_dirty--;
  880. count++;
  881. }
  882. } else if (buffer_uptodate(bh) && startio) {
  883. /*
  884. * we got here because the buffer is already mapped.
  885. * That means it must already have extents allocated
  886. * underneath it. Map the extent by reading it.
  887. */
  888. if (!iomap_valid || type != 0) {
  889. flags = BMAPI_READ;
  890. size = xfs_probe_cluster(inode, page, bh,
  891. head, 1);
  892. err = xfs_map_blocks(inode, offset, size,
  893. &iomap, flags);
  894. if (err)
  895. goto error;
  896. iomap_valid = xfs_iomap_valid(&iomap, offset);
  897. }
  898. type = 0;
  899. if (!test_and_set_bit(BH_Lock, &bh->b_state)) {
  900. ASSERT(buffer_mapped(bh));
  901. if (iomap_valid)
  902. all_bh = 1;
  903. xfs_add_to_ioend(inode, bh, offset, type,
  904. &ioend, !iomap_valid);
  905. page_dirty--;
  906. count++;
  907. } else {
  908. iomap_valid = 0;
  909. }
  910. } else if ((buffer_uptodate(bh) || PageUptodate(page)) &&
  911. (unmapped || startio)) {
  912. iomap_valid = 0;
  913. }
  914. if (!iohead)
  915. iohead = ioend;
  916. } while (offset += len, ((bh = bh->b_this_page) != head));
  917. if (uptodate && bh == head)
  918. SetPageUptodate(page);
  919. if (startio)
  920. xfs_start_page_writeback(page, wbc, 1, count);
  921. if (ioend && iomap_valid) {
  922. offset = (iomap.iomap_offset + iomap.iomap_bsize - 1) >>
  923. PAGE_CACHE_SHIFT;
  924. tlast = min_t(pgoff_t, offset, last_index);
  925. xfs_cluster_write(inode, page->index + 1, &iomap, &ioend,
  926. wbc, startio, all_bh, tlast);
  927. }
  928. if (iohead)
  929. xfs_submit_ioend(iohead);
  930. return page_dirty;
  931. error:
  932. if (iohead)
  933. xfs_cancel_ioend(iohead);
  934. /*
  935. * If it's delalloc and we have nowhere to put it,
  936. * throw it away, unless the lower layers told
  937. * us to try again.
  938. */
  939. if (err != -EAGAIN) {
  940. if (!unmapped)
  941. block_invalidatepage(page, 0);
  942. ClearPageUptodate(page);
  943. }
  944. return err;
  945. }
  946. /*
  947. * writepage: Called from one of two places:
  948. *
  949. * 1. we are flushing a delalloc buffer head.
  950. *
  951. * 2. we are writing out a dirty page. Typically the page dirty
  952. * state is cleared before we get here. In this case is it
  953. * conceivable we have no buffer heads.
  954. *
  955. * For delalloc space on the page we need to allocate space and
  956. * flush it. For unmapped buffer heads on the page we should
  957. * allocate space if the page is uptodate. For any other dirty
  958. * buffer heads on the page we should flush them.
  959. *
  960. * If we detect that a transaction would be required to flush
  961. * the page, we have to check the process flags first, if we
  962. * are already in a transaction or disk I/O during allocations
  963. * is off, we need to fail the writepage and redirty the page.
  964. */
  965. STATIC int
  966. xfs_vm_writepage(
  967. struct page *page,
  968. struct writeback_control *wbc)
  969. {
  970. int error;
  971. int need_trans;
  972. int delalloc, unmapped, unwritten;
  973. struct inode *inode = page->mapping->host;
  974. xfs_page_trace(XFS_WRITEPAGE_ENTER, inode, page, 0);
  975. /*
  976. * We need a transaction if:
  977. * 1. There are delalloc buffers on the page
  978. * 2. The page is uptodate and we have unmapped buffers
  979. * 3. The page is uptodate and we have no buffers
  980. * 4. There are unwritten buffers on the page
  981. */
  982. if (!page_has_buffers(page)) {
  983. unmapped = 1;
  984. need_trans = 1;
  985. } else {
  986. xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
  987. if (!PageUptodate(page))
  988. unmapped = 0;
  989. need_trans = delalloc + unmapped + unwritten;
  990. }
  991. /*
  992. * If we need a transaction and the process flags say
  993. * we are already in a transaction, or no IO is allowed
  994. * then mark the page dirty again and leave the page
  995. * as is.
  996. */
  997. if (current_test_flags(PF_FSTRANS) && need_trans)
  998. goto out_fail;
  999. /*
  1000. * Delay hooking up buffer heads until we have
  1001. * made our go/no-go decision.
  1002. */
  1003. if (!page_has_buffers(page))
  1004. create_empty_buffers(page, 1 << inode->i_blkbits, 0);
  1005. /*
  1006. * Convert delayed allocate, unwritten or unmapped space
  1007. * to real space and flush out to disk.
  1008. */
  1009. error = xfs_page_state_convert(inode, page, wbc, 1, unmapped);
  1010. if (error == -EAGAIN)
  1011. goto out_fail;
  1012. if (unlikely(error < 0))
  1013. goto out_unlock;
  1014. return 0;
  1015. out_fail:
  1016. redirty_page_for_writepage(wbc, page);
  1017. unlock_page(page);
  1018. return 0;
  1019. out_unlock:
  1020. unlock_page(page);
  1021. return error;
  1022. }
  1023. STATIC int
  1024. xfs_vm_writepages(
  1025. struct address_space *mapping,
  1026. struct writeback_control *wbc)
  1027. {
  1028. struct bhv_vnode *vp = vn_from_inode(mapping->host);
  1029. if (VN_TRUNC(vp))
  1030. VUNTRUNCATE(vp);
  1031. return generic_writepages(mapping, wbc);
  1032. }
  1033. /*
  1034. * Called to move a page into cleanable state - and from there
  1035. * to be released. Possibly the page is already clean. We always
  1036. * have buffer heads in this call.
  1037. *
  1038. * Returns 0 if the page is ok to release, 1 otherwise.
  1039. *
  1040. * Possible scenarios are:
  1041. *
  1042. * 1. We are being called to release a page which has been written
  1043. * to via regular I/O. buffer heads will be dirty and possibly
  1044. * delalloc. If no delalloc buffer heads in this case then we
  1045. * can just return zero.
  1046. *
  1047. * 2. We are called to release a page which has been written via
  1048. * mmap, all we need to do is ensure there is no delalloc
  1049. * state in the buffer heads, if not we can let the caller
  1050. * free them and we should come back later via writepage.
  1051. */
  1052. STATIC int
  1053. xfs_vm_releasepage(
  1054. struct page *page,
  1055. gfp_t gfp_mask)
  1056. {
  1057. struct inode *inode = page->mapping->host;
  1058. int dirty, delalloc, unmapped, unwritten;
  1059. struct writeback_control wbc = {
  1060. .sync_mode = WB_SYNC_ALL,
  1061. .nr_to_write = 1,
  1062. };
  1063. xfs_page_trace(XFS_RELEASEPAGE_ENTER, inode, page, 0);
  1064. if (!page_has_buffers(page))
  1065. return 0;
  1066. xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
  1067. if (!delalloc && !unwritten)
  1068. goto free_buffers;
  1069. if (!(gfp_mask & __GFP_FS))
  1070. return 0;
  1071. /* If we are already inside a transaction or the thread cannot
  1072. * do I/O, we cannot release this page.
  1073. */
  1074. if (current_test_flags(PF_FSTRANS))
  1075. return 0;
  1076. /*
  1077. * Convert delalloc space to real space, do not flush the
  1078. * data out to disk, that will be done by the caller.
  1079. * Never need to allocate space here - we will always
  1080. * come back to writepage in that case.
  1081. */
  1082. dirty = xfs_page_state_convert(inode, page, &wbc, 0, 0);
  1083. if (dirty == 0 && !unwritten)
  1084. goto free_buffers;
  1085. return 0;
  1086. free_buffers:
  1087. return try_to_free_buffers(page);
  1088. }
  1089. STATIC int
  1090. __xfs_get_blocks(
  1091. struct inode *inode,
  1092. sector_t iblock,
  1093. struct buffer_head *bh_result,
  1094. int create,
  1095. int direct,
  1096. bmapi_flags_t flags)
  1097. {
  1098. bhv_vnode_t *vp = vn_from_inode(inode);
  1099. xfs_iomap_t iomap;
  1100. xfs_off_t offset;
  1101. ssize_t size;
  1102. int niomap = 1;
  1103. int error;
  1104. offset = (xfs_off_t)iblock << inode->i_blkbits;
  1105. ASSERT(bh_result->b_size >= (1 << inode->i_blkbits));
  1106. size = bh_result->b_size;
  1107. error = bhv_vop_bmap(vp, offset, size,
  1108. create ? flags : BMAPI_READ, &iomap, &niomap);
  1109. if (error)
  1110. return -error;
  1111. if (niomap == 0)
  1112. return 0;
  1113. if (iomap.iomap_bn != IOMAP_DADDR_NULL) {
  1114. /*
  1115. * For unwritten extents do not report a disk address on
  1116. * the read case (treat as if we're reading into a hole).
  1117. */
  1118. if (create || !(iomap.iomap_flags & IOMAP_UNWRITTEN)) {
  1119. xfs_map_buffer(bh_result, &iomap, offset,
  1120. inode->i_blkbits);
  1121. }
  1122. if (create && (iomap.iomap_flags & IOMAP_UNWRITTEN)) {
  1123. if (direct)
  1124. bh_result->b_private = inode;
  1125. set_buffer_unwritten(bh_result);
  1126. }
  1127. }
  1128. /*
  1129. * If this is a realtime file, data may be on a different device.
  1130. * to that pointed to from the buffer_head b_bdev currently.
  1131. */
  1132. bh_result->b_bdev = iomap.iomap_target->bt_bdev;
  1133. /*
  1134. * If we previously allocated a block out beyond eof and we are now
  1135. * coming back to use it then we will need to flag it as new even if it
  1136. * has a disk address.
  1137. *
  1138. * With sub-block writes into unwritten extents we also need to mark
  1139. * the buffer as new so that the unwritten parts of the buffer gets
  1140. * correctly zeroed.
  1141. */
  1142. if (create &&
  1143. ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
  1144. (offset >= i_size_read(inode)) ||
  1145. (iomap.iomap_flags & (IOMAP_NEW|IOMAP_UNWRITTEN))))
  1146. set_buffer_new(bh_result);
  1147. if (iomap.iomap_flags & IOMAP_DELAY) {
  1148. BUG_ON(direct);
  1149. if (create) {
  1150. set_buffer_uptodate(bh_result);
  1151. set_buffer_mapped(bh_result);
  1152. set_buffer_delay(bh_result);
  1153. }
  1154. }
  1155. if (direct || size > (1 << inode->i_blkbits)) {
  1156. ASSERT(iomap.iomap_bsize - iomap.iomap_delta > 0);
  1157. offset = min_t(xfs_off_t,
  1158. iomap.iomap_bsize - iomap.iomap_delta, size);
  1159. bh_result->b_size = (ssize_t)min_t(xfs_off_t, LONG_MAX, offset);
  1160. }
  1161. return 0;
  1162. }
  1163. int
  1164. xfs_get_blocks(
  1165. struct inode *inode,
  1166. sector_t iblock,
  1167. struct buffer_head *bh_result,
  1168. int create)
  1169. {
  1170. return __xfs_get_blocks(inode, iblock,
  1171. bh_result, create, 0, BMAPI_WRITE);
  1172. }
  1173. STATIC int
  1174. xfs_get_blocks_direct(
  1175. struct inode *inode,
  1176. sector_t iblock,
  1177. struct buffer_head *bh_result,
  1178. int create)
  1179. {
  1180. return __xfs_get_blocks(inode, iblock,
  1181. bh_result, create, 1, BMAPI_WRITE|BMAPI_DIRECT);
  1182. }
  1183. STATIC void
  1184. xfs_end_io_direct(
  1185. struct kiocb *iocb,
  1186. loff_t offset,
  1187. ssize_t size,
  1188. void *private)
  1189. {
  1190. xfs_ioend_t *ioend = iocb->private;
  1191. /*
  1192. * Non-NULL private data means we need to issue a transaction to
  1193. * convert a range from unwritten to written extents. This needs
  1194. * to happen from process context but aio+dio I/O completion
  1195. * happens from irq context so we need to defer it to a workqueue.
  1196. * This is not necessary for synchronous direct I/O, but we do
  1197. * it anyway to keep the code uniform and simpler.
  1198. *
  1199. * The core direct I/O code might be changed to always call the
  1200. * completion handler in the future, in which case all this can
  1201. * go away.
  1202. */
  1203. if (private && size > 0) {
  1204. ioend->io_offset = offset;
  1205. ioend->io_size = size;
  1206. xfs_finish_ioend(ioend);
  1207. } else {
  1208. xfs_destroy_ioend(ioend);
  1209. }
  1210. /*
  1211. * blockdev_direct_IO can return an error even after the I/O
  1212. * completion handler was called. Thus we need to protect
  1213. * against double-freeing.
  1214. */
  1215. iocb->private = NULL;
  1216. }
  1217. STATIC ssize_t
  1218. xfs_vm_direct_IO(
  1219. int rw,
  1220. struct kiocb *iocb,
  1221. const struct iovec *iov,
  1222. loff_t offset,
  1223. unsigned long nr_segs)
  1224. {
  1225. struct file *file = iocb->ki_filp;
  1226. struct inode *inode = file->f_mapping->host;
  1227. bhv_vnode_t *vp = vn_from_inode(inode);
  1228. xfs_iomap_t iomap;
  1229. int maps = 1;
  1230. int error;
  1231. ssize_t ret;
  1232. error = bhv_vop_bmap(vp, offset, 0, BMAPI_DEVICE, &iomap, &maps);
  1233. if (error)
  1234. return -error;
  1235. iocb->private = xfs_alloc_ioend(inode, IOMAP_UNWRITTEN);
  1236. if (rw == WRITE) {
  1237. ret = blockdev_direct_IO_own_locking(rw, iocb, inode,
  1238. iomap.iomap_target->bt_bdev,
  1239. iov, offset, nr_segs,
  1240. xfs_get_blocks_direct,
  1241. xfs_end_io_direct);
  1242. } else {
  1243. ret = blockdev_direct_IO_no_locking(rw, iocb, inode,
  1244. iomap.iomap_target->bt_bdev,
  1245. iov, offset, nr_segs,
  1246. xfs_get_blocks_direct,
  1247. xfs_end_io_direct);
  1248. }
  1249. if (unlikely(ret != -EIOCBQUEUED && iocb->private))
  1250. xfs_destroy_ioend(iocb->private);
  1251. return ret;
  1252. }
  1253. STATIC int
  1254. xfs_vm_prepare_write(
  1255. struct file *file,
  1256. struct page *page,
  1257. unsigned int from,
  1258. unsigned int to)
  1259. {
  1260. return block_prepare_write(page, from, to, xfs_get_blocks);
  1261. }
  1262. STATIC sector_t
  1263. xfs_vm_bmap(
  1264. struct address_space *mapping,
  1265. sector_t block)
  1266. {
  1267. struct inode *inode = (struct inode *)mapping->host;
  1268. bhv_vnode_t *vp = vn_from_inode(inode);
  1269. vn_trace_entry(vp, __FUNCTION__, (inst_t *)__return_address);
  1270. bhv_vop_rwlock(vp, VRWLOCK_READ);
  1271. bhv_vop_flush_pages(vp, (xfs_off_t)0, -1, 0, FI_REMAPF);
  1272. bhv_vop_rwunlock(vp, VRWLOCK_READ);
  1273. return generic_block_bmap(mapping, block, xfs_get_blocks);
  1274. }
  1275. STATIC int
  1276. xfs_vm_readpage(
  1277. struct file *unused,
  1278. struct page *page)
  1279. {
  1280. return mpage_readpage(page, xfs_get_blocks);
  1281. }
  1282. STATIC int
  1283. xfs_vm_readpages(
  1284. struct file *unused,
  1285. struct address_space *mapping,
  1286. struct list_head *pages,
  1287. unsigned nr_pages)
  1288. {
  1289. return mpage_readpages(mapping, pages, nr_pages, xfs_get_blocks);
  1290. }
  1291. STATIC void
  1292. xfs_vm_invalidatepage(
  1293. struct page *page,
  1294. unsigned long offset)
  1295. {
  1296. xfs_page_trace(XFS_INVALIDPAGE_ENTER,
  1297. page->mapping->host, page, offset);
  1298. block_invalidatepage(page, offset);
  1299. }
  1300. const struct address_space_operations xfs_address_space_operations = {
  1301. .readpage = xfs_vm_readpage,
  1302. .readpages = xfs_vm_readpages,
  1303. .writepage = xfs_vm_writepage,
  1304. .writepages = xfs_vm_writepages,
  1305. .sync_page = block_sync_page,
  1306. .releasepage = xfs_vm_releasepage,
  1307. .invalidatepage = xfs_vm_invalidatepage,
  1308. .prepare_write = xfs_vm_prepare_write,
  1309. .commit_write = generic_commit_write,
  1310. .bmap = xfs_vm_bmap,
  1311. .direct_IO = xfs_vm_direct_IO,
  1312. .migratepage = buffer_migrate_page,
  1313. };