direct-io.c 35 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256
  1. /*
  2. * fs/direct-io.c
  3. *
  4. * Copyright (C) 2002, Linus Torvalds.
  5. *
  6. * O_DIRECT
  7. *
  8. * 04Jul2002 akpm@zip.com.au
  9. * Initial version
  10. * 11Sep2002 janetinc@us.ibm.com
  11. * added readv/writev support.
  12. * 29Oct2002 akpm@zip.com.au
  13. * rewrote bio_add_page() support.
  14. * 30Oct2002 pbadari@us.ibm.com
  15. * added support for non-aligned IO.
  16. * 06Nov2002 pbadari@us.ibm.com
  17. * added asynchronous IO support.
  18. * 21Jul2003 nathans@sgi.com
  19. * added IO completion notifier.
  20. */
  21. #include <linux/kernel.h>
  22. #include <linux/module.h>
  23. #include <linux/types.h>
  24. #include <linux/fs.h>
  25. #include <linux/mm.h>
  26. #include <linux/slab.h>
  27. #include <linux/highmem.h>
  28. #include <linux/pagemap.h>
  29. #include <linux/task_io_accounting_ops.h>
  30. #include <linux/bio.h>
  31. #include <linux/wait.h>
  32. #include <linux/err.h>
  33. #include <linux/blkdev.h>
  34. #include <linux/buffer_head.h>
  35. #include <linux/rwsem.h>
  36. #include <linux/uio.h>
  37. #include <asm/atomic.h>
  38. /*
  39. * How many user pages to map in one call to get_user_pages(). This determines
  40. * the size of a structure on the stack.
  41. */
  42. #define DIO_PAGES 64
  43. /*
  44. * This code generally works in units of "dio_blocks". A dio_block is
  45. * somewhere between the hard sector size and the filesystem block size. it
  46. * is determined on a per-invocation basis. When talking to the filesystem
  47. * we need to convert dio_blocks to fs_blocks by scaling the dio_block quantity
  48. * down by dio->blkfactor. Similarly, fs-blocksize quantities are converted
  49. * to bio_block quantities by shifting left by blkfactor.
  50. *
  51. * If blkfactor is zero then the user's request was aligned to the filesystem's
  52. * blocksize.
  53. *
  54. * lock_type is DIO_LOCKING for regular files on direct-IO-naive filesystems.
  55. * This determines whether we need to do the fancy locking which prevents
  56. * direct-IO from being able to read uninitialised disk blocks. If its zero
  57. * (blockdev) this locking is not done, and if it is DIO_OWN_LOCKING i_mutex is
  58. * not held for the entire direct write (taken briefly, initially, during a
  59. * direct read though, but its never held for the duration of a direct-IO).
  60. */
  61. struct dio {
  62. /* BIO submission state */
  63. struct bio *bio; /* bio under assembly */
  64. struct inode *inode;
  65. int rw;
  66. loff_t i_size; /* i_size when submitted */
  67. int lock_type; /* doesn't change */
  68. unsigned blkbits; /* doesn't change */
  69. unsigned blkfactor; /* When we're using an alignment which
  70. is finer than the filesystem's soft
  71. blocksize, this specifies how much
  72. finer. blkfactor=2 means 1/4-block
  73. alignment. Does not change */
  74. unsigned start_zero_done; /* flag: sub-blocksize zeroing has
  75. been performed at the start of a
  76. write */
  77. int pages_in_io; /* approximate total IO pages */
  78. size_t size; /* total request size (doesn't change)*/
  79. sector_t block_in_file; /* Current offset into the underlying
  80. file in dio_block units. */
  81. unsigned blocks_available; /* At block_in_file. changes */
  82. sector_t final_block_in_request;/* doesn't change */
  83. unsigned first_block_in_page; /* doesn't change, Used only once */
  84. int boundary; /* prev block is at a boundary */
  85. int reap_counter; /* rate limit reaping */
  86. get_block_t *get_block; /* block mapping function */
  87. dio_iodone_t *end_io; /* IO completion function */
  88. sector_t final_block_in_bio; /* current final block in bio + 1 */
  89. sector_t next_block_for_io; /* next block to be put under IO,
  90. in dio_blocks units */
  91. struct buffer_head map_bh; /* last get_block() result */
  92. /*
  93. * Deferred addition of a page to the dio. These variables are
  94. * private to dio_send_cur_page(), submit_page_section() and
  95. * dio_bio_add_page().
  96. */
  97. struct page *cur_page; /* The page */
  98. unsigned cur_page_offset; /* Offset into it, in bytes */
  99. unsigned cur_page_len; /* Nr of bytes at cur_page_offset */
  100. sector_t cur_page_block; /* Where it starts */
  101. /*
  102. * Page fetching state. These variables belong to dio_refill_pages().
  103. */
  104. int curr_page; /* changes */
  105. int total_pages; /* doesn't change */
  106. unsigned long curr_user_address;/* changes */
  107. /*
  108. * Page queue. These variables belong to dio_refill_pages() and
  109. * dio_get_page().
  110. */
  111. struct page *pages[DIO_PAGES]; /* page buffer */
  112. unsigned head; /* next page to process */
  113. unsigned tail; /* last valid page + 1 */
  114. int page_errors; /* errno from get_user_pages() */
  115. /* BIO completion state */
  116. spinlock_t bio_lock; /* protects BIO fields below */
  117. unsigned long refcount; /* direct_io_worker() and bios */
  118. struct bio *bio_list; /* singly linked via bi_private */
  119. struct task_struct *waiter; /* waiting task (NULL if none) */
  120. /* AIO related stuff */
  121. struct kiocb *iocb; /* kiocb */
  122. int is_async; /* is IO async ? */
  123. int io_error; /* IO error in completion path */
  124. ssize_t result; /* IO result */
  125. };
  126. /*
  127. * How many pages are in the queue?
  128. */
  129. static inline unsigned dio_pages_present(struct dio *dio)
  130. {
  131. return dio->tail - dio->head;
  132. }
  133. /*
  134. * Go grab and pin some userspace pages. Typically we'll get 64 at a time.
  135. */
  136. static int dio_refill_pages(struct dio *dio)
  137. {
  138. int ret;
  139. int nr_pages;
  140. nr_pages = min(dio->total_pages - dio->curr_page, DIO_PAGES);
  141. down_read(&current->mm->mmap_sem);
  142. ret = get_user_pages(
  143. current, /* Task for fault acounting */
  144. current->mm, /* whose pages? */
  145. dio->curr_user_address, /* Where from? */
  146. nr_pages, /* How many pages? */
  147. dio->rw == READ, /* Write to memory? */
  148. 0, /* force (?) */
  149. &dio->pages[0],
  150. NULL); /* vmas */
  151. up_read(&current->mm->mmap_sem);
  152. if (ret < 0 && dio->blocks_available && (dio->rw & WRITE)) {
  153. struct page *page = ZERO_PAGE(dio->curr_user_address);
  154. /*
  155. * A memory fault, but the filesystem has some outstanding
  156. * mapped blocks. We need to use those blocks up to avoid
  157. * leaking stale data in the file.
  158. */
  159. if (dio->page_errors == 0)
  160. dio->page_errors = ret;
  161. page_cache_get(page);
  162. dio->pages[0] = page;
  163. dio->head = 0;
  164. dio->tail = 1;
  165. ret = 0;
  166. goto out;
  167. }
  168. if (ret >= 0) {
  169. dio->curr_user_address += ret * PAGE_SIZE;
  170. dio->curr_page += ret;
  171. dio->head = 0;
  172. dio->tail = ret;
  173. ret = 0;
  174. }
  175. out:
  176. return ret;
  177. }
  178. /*
  179. * Get another userspace page. Returns an ERR_PTR on error. Pages are
  180. * buffered inside the dio so that we can call get_user_pages() against a
  181. * decent number of pages, less frequently. To provide nicer use of the
  182. * L1 cache.
  183. */
  184. static struct page *dio_get_page(struct dio *dio)
  185. {
  186. if (dio_pages_present(dio) == 0) {
  187. int ret;
  188. ret = dio_refill_pages(dio);
  189. if (ret)
  190. return ERR_PTR(ret);
  191. BUG_ON(dio_pages_present(dio) == 0);
  192. }
  193. return dio->pages[dio->head++];
  194. }
  195. /**
  196. * dio_complete() - called when all DIO BIO I/O has been completed
  197. * @offset: the byte offset in the file of the completed operation
  198. *
  199. * This releases locks as dictated by the locking type, lets interested parties
  200. * know that a DIO operation has completed, and calculates the resulting return
  201. * code for the operation.
  202. *
  203. * It lets the filesystem know if it registered an interest earlier via
  204. * get_block. Pass the private field of the map buffer_head so that
  205. * filesystems can use it to hold additional state between get_block calls and
  206. * dio_complete.
  207. */
  208. static int dio_complete(struct dio *dio, loff_t offset, int ret)
  209. {
  210. ssize_t transferred = 0;
  211. /*
  212. * AIO submission can race with bio completion to get here while
  213. * expecting to have the last io completed by bio completion.
  214. * In that case -EIOCBQUEUED is in fact not an error we want
  215. * to preserve through this call.
  216. */
  217. if (ret == -EIOCBQUEUED)
  218. ret = 0;
  219. if (dio->result) {
  220. transferred = dio->result;
  221. /* Check for short read case */
  222. if ((dio->rw == READ) && ((offset + transferred) > dio->i_size))
  223. transferred = dio->i_size - offset;
  224. }
  225. if (dio->end_io && dio->result)
  226. dio->end_io(dio->iocb, offset, transferred,
  227. dio->map_bh.b_private);
  228. if (dio->lock_type == DIO_LOCKING)
  229. /* lockdep: non-owner release */
  230. up_read_non_owner(&dio->inode->i_alloc_sem);
  231. if (ret == 0)
  232. ret = dio->page_errors;
  233. if (ret == 0)
  234. ret = dio->io_error;
  235. if (ret == 0)
  236. ret = transferred;
  237. return ret;
  238. }
  239. static int dio_bio_complete(struct dio *dio, struct bio *bio);
  240. /*
  241. * Asynchronous IO callback.
  242. */
  243. static int dio_bio_end_aio(struct bio *bio, unsigned int bytes_done, int error)
  244. {
  245. struct dio *dio = bio->bi_private;
  246. unsigned long remaining;
  247. unsigned long flags;
  248. if (bio->bi_size)
  249. return 1;
  250. /* cleanup the bio */
  251. dio_bio_complete(dio, bio);
  252. spin_lock_irqsave(&dio->bio_lock, flags);
  253. remaining = --dio->refcount;
  254. if (remaining == 1 && dio->waiter)
  255. wake_up_process(dio->waiter);
  256. spin_unlock_irqrestore(&dio->bio_lock, flags);
  257. if (remaining == 0) {
  258. int ret = dio_complete(dio, dio->iocb->ki_pos, 0);
  259. aio_complete(dio->iocb, ret, 0);
  260. kfree(dio);
  261. }
  262. return 0;
  263. }
  264. /*
  265. * The BIO completion handler simply queues the BIO up for the process-context
  266. * handler.
  267. *
  268. * During I/O bi_private points at the dio. After I/O, bi_private is used to
  269. * implement a singly-linked list of completed BIOs, at dio->bio_list.
  270. */
  271. static int dio_bio_end_io(struct bio *bio, unsigned int bytes_done, int error)
  272. {
  273. struct dio *dio = bio->bi_private;
  274. unsigned long flags;
  275. if (bio->bi_size)
  276. return 1;
  277. spin_lock_irqsave(&dio->bio_lock, flags);
  278. bio->bi_private = dio->bio_list;
  279. dio->bio_list = bio;
  280. if (--dio->refcount == 1 && dio->waiter)
  281. wake_up_process(dio->waiter);
  282. spin_unlock_irqrestore(&dio->bio_lock, flags);
  283. return 0;
  284. }
  285. static int
  286. dio_bio_alloc(struct dio *dio, struct block_device *bdev,
  287. sector_t first_sector, int nr_vecs)
  288. {
  289. struct bio *bio;
  290. bio = bio_alloc(GFP_KERNEL, nr_vecs);
  291. if (bio == NULL)
  292. return -ENOMEM;
  293. bio->bi_bdev = bdev;
  294. bio->bi_sector = first_sector;
  295. if (dio->is_async)
  296. bio->bi_end_io = dio_bio_end_aio;
  297. else
  298. bio->bi_end_io = dio_bio_end_io;
  299. dio->bio = bio;
  300. return 0;
  301. }
  302. /*
  303. * In the AIO read case we speculatively dirty the pages before starting IO.
  304. * During IO completion, any of these pages which happen to have been written
  305. * back will be redirtied by bio_check_pages_dirty().
  306. *
  307. * bios hold a dio reference between submit_bio and ->end_io.
  308. */
  309. static void dio_bio_submit(struct dio *dio)
  310. {
  311. struct bio *bio = dio->bio;
  312. unsigned long flags;
  313. bio->bi_private = dio;
  314. spin_lock_irqsave(&dio->bio_lock, flags);
  315. dio->refcount++;
  316. spin_unlock_irqrestore(&dio->bio_lock, flags);
  317. if (dio->is_async && dio->rw == READ)
  318. bio_set_pages_dirty(bio);
  319. submit_bio(dio->rw, bio);
  320. dio->bio = NULL;
  321. dio->boundary = 0;
  322. }
  323. /*
  324. * Release any resources in case of a failure
  325. */
  326. static void dio_cleanup(struct dio *dio)
  327. {
  328. while (dio_pages_present(dio))
  329. page_cache_release(dio_get_page(dio));
  330. }
  331. /*
  332. * Wait for the next BIO to complete. Remove it and return it. NULL is
  333. * returned once all BIOs have been completed. This must only be called once
  334. * all bios have been issued so that dio->refcount can only decrease. This
  335. * requires that that the caller hold a reference on the dio.
  336. */
  337. static struct bio *dio_await_one(struct dio *dio)
  338. {
  339. unsigned long flags;
  340. struct bio *bio = NULL;
  341. spin_lock_irqsave(&dio->bio_lock, flags);
  342. /*
  343. * Wait as long as the list is empty and there are bios in flight. bio
  344. * completion drops the count, maybe adds to the list, and wakes while
  345. * holding the bio_lock so we don't need set_current_state()'s barrier
  346. * and can call it after testing our condition.
  347. */
  348. while (dio->refcount > 1 && dio->bio_list == NULL) {
  349. __set_current_state(TASK_UNINTERRUPTIBLE);
  350. dio->waiter = current;
  351. spin_unlock_irqrestore(&dio->bio_lock, flags);
  352. io_schedule();
  353. /* wake up sets us TASK_RUNNING */
  354. spin_lock_irqsave(&dio->bio_lock, flags);
  355. dio->waiter = NULL;
  356. }
  357. if (dio->bio_list) {
  358. bio = dio->bio_list;
  359. dio->bio_list = bio->bi_private;
  360. }
  361. spin_unlock_irqrestore(&dio->bio_lock, flags);
  362. return bio;
  363. }
  364. /*
  365. * Process one completed BIO. No locks are held.
  366. */
  367. static int dio_bio_complete(struct dio *dio, struct bio *bio)
  368. {
  369. const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
  370. struct bio_vec *bvec = bio->bi_io_vec;
  371. int page_no;
  372. if (!uptodate)
  373. dio->io_error = -EIO;
  374. if (dio->is_async && dio->rw == READ) {
  375. bio_check_pages_dirty(bio); /* transfers ownership */
  376. } else {
  377. for (page_no = 0; page_no < bio->bi_vcnt; page_no++) {
  378. struct page *page = bvec[page_no].bv_page;
  379. if (dio->rw == READ && !PageCompound(page))
  380. set_page_dirty_lock(page);
  381. page_cache_release(page);
  382. }
  383. bio_put(bio);
  384. }
  385. return uptodate ? 0 : -EIO;
  386. }
  387. /*
  388. * Wait on and process all in-flight BIOs. This must only be called once
  389. * all bios have been issued so that the refcount can only decrease.
  390. * This just waits for all bios to make it through dio_bio_complete. IO
  391. * errors are propogated through dio->io_error and should be propogated via
  392. * dio_complete().
  393. */
  394. static void dio_await_completion(struct dio *dio)
  395. {
  396. struct bio *bio;
  397. do {
  398. bio = dio_await_one(dio);
  399. if (bio)
  400. dio_bio_complete(dio, bio);
  401. } while (bio);
  402. }
  403. /*
  404. * A really large O_DIRECT read or write can generate a lot of BIOs. So
  405. * to keep the memory consumption sane we periodically reap any completed BIOs
  406. * during the BIO generation phase.
  407. *
  408. * This also helps to limit the peak amount of pinned userspace memory.
  409. */
  410. static int dio_bio_reap(struct dio *dio)
  411. {
  412. int ret = 0;
  413. if (dio->reap_counter++ >= 64) {
  414. while (dio->bio_list) {
  415. unsigned long flags;
  416. struct bio *bio;
  417. int ret2;
  418. spin_lock_irqsave(&dio->bio_lock, flags);
  419. bio = dio->bio_list;
  420. dio->bio_list = bio->bi_private;
  421. spin_unlock_irqrestore(&dio->bio_lock, flags);
  422. ret2 = dio_bio_complete(dio, bio);
  423. if (ret == 0)
  424. ret = ret2;
  425. }
  426. dio->reap_counter = 0;
  427. }
  428. return ret;
  429. }
  430. /*
  431. * Call into the fs to map some more disk blocks. We record the current number
  432. * of available blocks at dio->blocks_available. These are in units of the
  433. * fs blocksize, (1 << inode->i_blkbits).
  434. *
  435. * The fs is allowed to map lots of blocks at once. If it wants to do that,
  436. * it uses the passed inode-relative block number as the file offset, as usual.
  437. *
  438. * get_block() is passed the number of i_blkbits-sized blocks which direct_io
  439. * has remaining to do. The fs should not map more than this number of blocks.
  440. *
  441. * If the fs has mapped a lot of blocks, it should populate bh->b_size to
  442. * indicate how much contiguous disk space has been made available at
  443. * bh->b_blocknr.
  444. *
  445. * If *any* of the mapped blocks are new, then the fs must set buffer_new().
  446. * This isn't very efficient...
  447. *
  448. * In the case of filesystem holes: the fs may return an arbitrarily-large
  449. * hole by returning an appropriate value in b_size and by clearing
  450. * buffer_mapped(). However the direct-io code will only process holes one
  451. * block at a time - it will repeatedly call get_block() as it walks the hole.
  452. */
  453. static int get_more_blocks(struct dio *dio)
  454. {
  455. int ret;
  456. struct buffer_head *map_bh = &dio->map_bh;
  457. sector_t fs_startblk; /* Into file, in filesystem-sized blocks */
  458. unsigned long fs_count; /* Number of filesystem-sized blocks */
  459. unsigned long dio_count;/* Number of dio_block-sized blocks */
  460. unsigned long blkmask;
  461. int create;
  462. /*
  463. * If there was a memory error and we've overwritten all the
  464. * mapped blocks then we can now return that memory error
  465. */
  466. ret = dio->page_errors;
  467. if (ret == 0) {
  468. BUG_ON(dio->block_in_file >= dio->final_block_in_request);
  469. fs_startblk = dio->block_in_file >> dio->blkfactor;
  470. dio_count = dio->final_block_in_request - dio->block_in_file;
  471. fs_count = dio_count >> dio->blkfactor;
  472. blkmask = (1 << dio->blkfactor) - 1;
  473. if (dio_count & blkmask)
  474. fs_count++;
  475. map_bh->b_state = 0;
  476. map_bh->b_size = fs_count << dio->inode->i_blkbits;
  477. create = dio->rw & WRITE;
  478. if (dio->lock_type == DIO_LOCKING) {
  479. if (dio->block_in_file < (i_size_read(dio->inode) >>
  480. dio->blkbits))
  481. create = 0;
  482. } else if (dio->lock_type == DIO_NO_LOCKING) {
  483. create = 0;
  484. }
  485. /*
  486. * For writes inside i_size we forbid block creations: only
  487. * overwrites are permitted. We fall back to buffered writes
  488. * at a higher level for inside-i_size block-instantiating
  489. * writes.
  490. */
  491. ret = (*dio->get_block)(dio->inode, fs_startblk,
  492. map_bh, create);
  493. }
  494. return ret;
  495. }
  496. /*
  497. * There is no bio. Make one now.
  498. */
  499. static int dio_new_bio(struct dio *dio, sector_t start_sector)
  500. {
  501. sector_t sector;
  502. int ret, nr_pages;
  503. ret = dio_bio_reap(dio);
  504. if (ret)
  505. goto out;
  506. sector = start_sector << (dio->blkbits - 9);
  507. nr_pages = min(dio->pages_in_io, bio_get_nr_vecs(dio->map_bh.b_bdev));
  508. BUG_ON(nr_pages <= 0);
  509. ret = dio_bio_alloc(dio, dio->map_bh.b_bdev, sector, nr_pages);
  510. dio->boundary = 0;
  511. out:
  512. return ret;
  513. }
  514. /*
  515. * Attempt to put the current chunk of 'cur_page' into the current BIO. If
  516. * that was successful then update final_block_in_bio and take a ref against
  517. * the just-added page.
  518. *
  519. * Return zero on success. Non-zero means the caller needs to start a new BIO.
  520. */
  521. static int dio_bio_add_page(struct dio *dio)
  522. {
  523. int ret;
  524. ret = bio_add_page(dio->bio, dio->cur_page,
  525. dio->cur_page_len, dio->cur_page_offset);
  526. if (ret == dio->cur_page_len) {
  527. /*
  528. * Decrement count only, if we are done with this page
  529. */
  530. if ((dio->cur_page_len + dio->cur_page_offset) == PAGE_SIZE)
  531. dio->pages_in_io--;
  532. page_cache_get(dio->cur_page);
  533. dio->final_block_in_bio = dio->cur_page_block +
  534. (dio->cur_page_len >> dio->blkbits);
  535. ret = 0;
  536. } else {
  537. ret = 1;
  538. }
  539. return ret;
  540. }
  541. /*
  542. * Put cur_page under IO. The section of cur_page which is described by
  543. * cur_page_offset,cur_page_len is put into a BIO. The section of cur_page
  544. * starts on-disk at cur_page_block.
  545. *
  546. * We take a ref against the page here (on behalf of its presence in the bio).
  547. *
  548. * The caller of this function is responsible for removing cur_page from the
  549. * dio, and for dropping the refcount which came from that presence.
  550. */
  551. static int dio_send_cur_page(struct dio *dio)
  552. {
  553. int ret = 0;
  554. if (dio->bio) {
  555. /*
  556. * See whether this new request is contiguous with the old
  557. */
  558. if (dio->final_block_in_bio != dio->cur_page_block)
  559. dio_bio_submit(dio);
  560. /*
  561. * Submit now if the underlying fs is about to perform a
  562. * metadata read
  563. */
  564. if (dio->boundary)
  565. dio_bio_submit(dio);
  566. }
  567. if (dio->bio == NULL) {
  568. ret = dio_new_bio(dio, dio->cur_page_block);
  569. if (ret)
  570. goto out;
  571. }
  572. if (dio_bio_add_page(dio) != 0) {
  573. dio_bio_submit(dio);
  574. ret = dio_new_bio(dio, dio->cur_page_block);
  575. if (ret == 0) {
  576. ret = dio_bio_add_page(dio);
  577. BUG_ON(ret != 0);
  578. }
  579. }
  580. out:
  581. return ret;
  582. }
  583. /*
  584. * An autonomous function to put a chunk of a page under deferred IO.
  585. *
  586. * The caller doesn't actually know (or care) whether this piece of page is in
  587. * a BIO, or is under IO or whatever. We just take care of all possible
  588. * situations here. The separation between the logic of do_direct_IO() and
  589. * that of submit_page_section() is important for clarity. Please don't break.
  590. *
  591. * The chunk of page starts on-disk at blocknr.
  592. *
  593. * We perform deferred IO, by recording the last-submitted page inside our
  594. * private part of the dio structure. If possible, we just expand the IO
  595. * across that page here.
  596. *
  597. * If that doesn't work out then we put the old page into the bio and add this
  598. * page to the dio instead.
  599. */
  600. static int
  601. submit_page_section(struct dio *dio, struct page *page,
  602. unsigned offset, unsigned len, sector_t blocknr)
  603. {
  604. int ret = 0;
  605. if (dio->rw & WRITE) {
  606. /*
  607. * Read accounting is performed in submit_bio()
  608. */
  609. task_io_account_write(len);
  610. }
  611. /*
  612. * Can we just grow the current page's presence in the dio?
  613. */
  614. if ( (dio->cur_page == page) &&
  615. (dio->cur_page_offset + dio->cur_page_len == offset) &&
  616. (dio->cur_page_block +
  617. (dio->cur_page_len >> dio->blkbits) == blocknr)) {
  618. dio->cur_page_len += len;
  619. /*
  620. * If dio->boundary then we want to schedule the IO now to
  621. * avoid metadata seeks.
  622. */
  623. if (dio->boundary) {
  624. ret = dio_send_cur_page(dio);
  625. page_cache_release(dio->cur_page);
  626. dio->cur_page = NULL;
  627. }
  628. goto out;
  629. }
  630. /*
  631. * If there's a deferred page already there then send it.
  632. */
  633. if (dio->cur_page) {
  634. ret = dio_send_cur_page(dio);
  635. page_cache_release(dio->cur_page);
  636. dio->cur_page = NULL;
  637. if (ret)
  638. goto out;
  639. }
  640. page_cache_get(page); /* It is in dio */
  641. dio->cur_page = page;
  642. dio->cur_page_offset = offset;
  643. dio->cur_page_len = len;
  644. dio->cur_page_block = blocknr;
  645. out:
  646. return ret;
  647. }
  648. /*
  649. * Clean any dirty buffers in the blockdev mapping which alias newly-created
  650. * file blocks. Only called for S_ISREG files - blockdevs do not set
  651. * buffer_new
  652. */
  653. static void clean_blockdev_aliases(struct dio *dio)
  654. {
  655. unsigned i;
  656. unsigned nblocks;
  657. nblocks = dio->map_bh.b_size >> dio->inode->i_blkbits;
  658. for (i = 0; i < nblocks; i++) {
  659. unmap_underlying_metadata(dio->map_bh.b_bdev,
  660. dio->map_bh.b_blocknr + i);
  661. }
  662. }
  663. /*
  664. * If we are not writing the entire block and get_block() allocated
  665. * the block for us, we need to fill-in the unused portion of the
  666. * block with zeros. This happens only if user-buffer, fileoffset or
  667. * io length is not filesystem block-size multiple.
  668. *
  669. * `end' is zero if we're doing the start of the IO, 1 at the end of the
  670. * IO.
  671. */
  672. static void dio_zero_block(struct dio *dio, int end)
  673. {
  674. unsigned dio_blocks_per_fs_block;
  675. unsigned this_chunk_blocks; /* In dio_blocks */
  676. unsigned this_chunk_bytes;
  677. struct page *page;
  678. dio->start_zero_done = 1;
  679. if (!dio->blkfactor || !buffer_new(&dio->map_bh))
  680. return;
  681. dio_blocks_per_fs_block = 1 << dio->blkfactor;
  682. this_chunk_blocks = dio->block_in_file & (dio_blocks_per_fs_block - 1);
  683. if (!this_chunk_blocks)
  684. return;
  685. /*
  686. * We need to zero out part of an fs block. It is either at the
  687. * beginning or the end of the fs block.
  688. */
  689. if (end)
  690. this_chunk_blocks = dio_blocks_per_fs_block - this_chunk_blocks;
  691. this_chunk_bytes = this_chunk_blocks << dio->blkbits;
  692. page = ZERO_PAGE(dio->curr_user_address);
  693. if (submit_page_section(dio, page, 0, this_chunk_bytes,
  694. dio->next_block_for_io))
  695. return;
  696. dio->next_block_for_io += this_chunk_blocks;
  697. }
  698. /*
  699. * Walk the user pages, and the file, mapping blocks to disk and generating
  700. * a sequence of (page,offset,len,block) mappings. These mappings are injected
  701. * into submit_page_section(), which takes care of the next stage of submission
  702. *
  703. * Direct IO against a blockdev is different from a file. Because we can
  704. * happily perform page-sized but 512-byte aligned IOs. It is important that
  705. * blockdev IO be able to have fine alignment and large sizes.
  706. *
  707. * So what we do is to permit the ->get_block function to populate bh.b_size
  708. * with the size of IO which is permitted at this offset and this i_blkbits.
  709. *
  710. * For best results, the blockdev should be set up with 512-byte i_blkbits and
  711. * it should set b_size to PAGE_SIZE or more inside get_block(). This gives
  712. * fine alignment but still allows this function to work in PAGE_SIZE units.
  713. */
  714. static int do_direct_IO(struct dio *dio)
  715. {
  716. const unsigned blkbits = dio->blkbits;
  717. const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
  718. struct page *page;
  719. unsigned block_in_page;
  720. struct buffer_head *map_bh = &dio->map_bh;
  721. int ret = 0;
  722. /* The I/O can start at any block offset within the first page */
  723. block_in_page = dio->first_block_in_page;
  724. while (dio->block_in_file < dio->final_block_in_request) {
  725. page = dio_get_page(dio);
  726. if (IS_ERR(page)) {
  727. ret = PTR_ERR(page);
  728. goto out;
  729. }
  730. while (block_in_page < blocks_per_page) {
  731. unsigned offset_in_page = block_in_page << blkbits;
  732. unsigned this_chunk_bytes; /* # of bytes mapped */
  733. unsigned this_chunk_blocks; /* # of blocks */
  734. unsigned u;
  735. if (dio->blocks_available == 0) {
  736. /*
  737. * Need to go and map some more disk
  738. */
  739. unsigned long blkmask;
  740. unsigned long dio_remainder;
  741. ret = get_more_blocks(dio);
  742. if (ret) {
  743. page_cache_release(page);
  744. goto out;
  745. }
  746. if (!buffer_mapped(map_bh))
  747. goto do_holes;
  748. dio->blocks_available =
  749. map_bh->b_size >> dio->blkbits;
  750. dio->next_block_for_io =
  751. map_bh->b_blocknr << dio->blkfactor;
  752. if (buffer_new(map_bh))
  753. clean_blockdev_aliases(dio);
  754. if (!dio->blkfactor)
  755. goto do_holes;
  756. blkmask = (1 << dio->blkfactor) - 1;
  757. dio_remainder = (dio->block_in_file & blkmask);
  758. /*
  759. * If we are at the start of IO and that IO
  760. * starts partway into a fs-block,
  761. * dio_remainder will be non-zero. If the IO
  762. * is a read then we can simply advance the IO
  763. * cursor to the first block which is to be
  764. * read. But if the IO is a write and the
  765. * block was newly allocated we cannot do that;
  766. * the start of the fs block must be zeroed out
  767. * on-disk
  768. */
  769. if (!buffer_new(map_bh))
  770. dio->next_block_for_io += dio_remainder;
  771. dio->blocks_available -= dio_remainder;
  772. }
  773. do_holes:
  774. /* Handle holes */
  775. if (!buffer_mapped(map_bh)) {
  776. char *kaddr;
  777. loff_t i_size_aligned;
  778. /* AKPM: eargh, -ENOTBLK is a hack */
  779. if (dio->rw & WRITE) {
  780. page_cache_release(page);
  781. return -ENOTBLK;
  782. }
  783. /*
  784. * Be sure to account for a partial block as the
  785. * last block in the file
  786. */
  787. i_size_aligned = ALIGN(i_size_read(dio->inode),
  788. 1 << blkbits);
  789. if (dio->block_in_file >=
  790. i_size_aligned >> blkbits) {
  791. /* We hit eof */
  792. page_cache_release(page);
  793. goto out;
  794. }
  795. kaddr = kmap_atomic(page, KM_USER0);
  796. memset(kaddr + (block_in_page << blkbits),
  797. 0, 1 << blkbits);
  798. flush_dcache_page(page);
  799. kunmap_atomic(kaddr, KM_USER0);
  800. dio->block_in_file++;
  801. block_in_page++;
  802. goto next_block;
  803. }
  804. /*
  805. * If we're performing IO which has an alignment which
  806. * is finer than the underlying fs, go check to see if
  807. * we must zero out the start of this block.
  808. */
  809. if (unlikely(dio->blkfactor && !dio->start_zero_done))
  810. dio_zero_block(dio, 0);
  811. /*
  812. * Work out, in this_chunk_blocks, how much disk we
  813. * can add to this page
  814. */
  815. this_chunk_blocks = dio->blocks_available;
  816. u = (PAGE_SIZE - offset_in_page) >> blkbits;
  817. if (this_chunk_blocks > u)
  818. this_chunk_blocks = u;
  819. u = dio->final_block_in_request - dio->block_in_file;
  820. if (this_chunk_blocks > u)
  821. this_chunk_blocks = u;
  822. this_chunk_bytes = this_chunk_blocks << blkbits;
  823. BUG_ON(this_chunk_bytes == 0);
  824. dio->boundary = buffer_boundary(map_bh);
  825. ret = submit_page_section(dio, page, offset_in_page,
  826. this_chunk_bytes, dio->next_block_for_io);
  827. if (ret) {
  828. page_cache_release(page);
  829. goto out;
  830. }
  831. dio->next_block_for_io += this_chunk_blocks;
  832. dio->block_in_file += this_chunk_blocks;
  833. block_in_page += this_chunk_blocks;
  834. dio->blocks_available -= this_chunk_blocks;
  835. next_block:
  836. BUG_ON(dio->block_in_file > dio->final_block_in_request);
  837. if (dio->block_in_file == dio->final_block_in_request)
  838. break;
  839. }
  840. /* Drop the ref which was taken in get_user_pages() */
  841. page_cache_release(page);
  842. block_in_page = 0;
  843. }
  844. out:
  845. return ret;
  846. }
  847. /*
  848. * Releases both i_mutex and i_alloc_sem
  849. */
  850. static ssize_t
  851. direct_io_worker(int rw, struct kiocb *iocb, struct inode *inode,
  852. const struct iovec *iov, loff_t offset, unsigned long nr_segs,
  853. unsigned blkbits, get_block_t get_block, dio_iodone_t end_io,
  854. struct dio *dio)
  855. {
  856. unsigned long user_addr;
  857. unsigned long flags;
  858. int seg;
  859. ssize_t ret = 0;
  860. ssize_t ret2;
  861. size_t bytes;
  862. dio->bio = NULL;
  863. dio->inode = inode;
  864. dio->rw = rw;
  865. dio->blkbits = blkbits;
  866. dio->blkfactor = inode->i_blkbits - blkbits;
  867. dio->start_zero_done = 0;
  868. dio->size = 0;
  869. dio->block_in_file = offset >> blkbits;
  870. dio->blocks_available = 0;
  871. dio->cur_page = NULL;
  872. dio->boundary = 0;
  873. dio->reap_counter = 0;
  874. dio->get_block = get_block;
  875. dio->end_io = end_io;
  876. dio->map_bh.b_private = NULL;
  877. dio->final_block_in_bio = -1;
  878. dio->next_block_for_io = -1;
  879. dio->page_errors = 0;
  880. dio->io_error = 0;
  881. dio->result = 0;
  882. dio->iocb = iocb;
  883. dio->i_size = i_size_read(inode);
  884. spin_lock_init(&dio->bio_lock);
  885. dio->refcount = 1;
  886. dio->bio_list = NULL;
  887. dio->waiter = NULL;
  888. /*
  889. * In case of non-aligned buffers, we may need 2 more
  890. * pages since we need to zero out first and last block.
  891. */
  892. if (unlikely(dio->blkfactor))
  893. dio->pages_in_io = 2;
  894. else
  895. dio->pages_in_io = 0;
  896. for (seg = 0; seg < nr_segs; seg++) {
  897. user_addr = (unsigned long)iov[seg].iov_base;
  898. dio->pages_in_io +=
  899. ((user_addr+iov[seg].iov_len +PAGE_SIZE-1)/PAGE_SIZE
  900. - user_addr/PAGE_SIZE);
  901. }
  902. for (seg = 0; seg < nr_segs; seg++) {
  903. user_addr = (unsigned long)iov[seg].iov_base;
  904. dio->size += bytes = iov[seg].iov_len;
  905. /* Index into the first page of the first block */
  906. dio->first_block_in_page = (user_addr & ~PAGE_MASK) >> blkbits;
  907. dio->final_block_in_request = dio->block_in_file +
  908. (bytes >> blkbits);
  909. /* Page fetching state */
  910. dio->head = 0;
  911. dio->tail = 0;
  912. dio->curr_page = 0;
  913. dio->total_pages = 0;
  914. if (user_addr & (PAGE_SIZE-1)) {
  915. dio->total_pages++;
  916. bytes -= PAGE_SIZE - (user_addr & (PAGE_SIZE - 1));
  917. }
  918. dio->total_pages += (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
  919. dio->curr_user_address = user_addr;
  920. ret = do_direct_IO(dio);
  921. dio->result += iov[seg].iov_len -
  922. ((dio->final_block_in_request - dio->block_in_file) <<
  923. blkbits);
  924. if (ret) {
  925. dio_cleanup(dio);
  926. break;
  927. }
  928. } /* end iovec loop */
  929. if (ret == -ENOTBLK && (rw & WRITE)) {
  930. /*
  931. * The remaining part of the request will be
  932. * be handled by buffered I/O when we return
  933. */
  934. ret = 0;
  935. }
  936. /*
  937. * There may be some unwritten disk at the end of a part-written
  938. * fs-block-sized block. Go zero that now.
  939. */
  940. dio_zero_block(dio, 1);
  941. if (dio->cur_page) {
  942. ret2 = dio_send_cur_page(dio);
  943. if (ret == 0)
  944. ret = ret2;
  945. page_cache_release(dio->cur_page);
  946. dio->cur_page = NULL;
  947. }
  948. if (dio->bio)
  949. dio_bio_submit(dio);
  950. /* All IO is now issued, send it on its way */
  951. blk_run_address_space(inode->i_mapping);
  952. /*
  953. * It is possible that, we return short IO due to end of file.
  954. * In that case, we need to release all the pages we got hold on.
  955. */
  956. dio_cleanup(dio);
  957. /*
  958. * All block lookups have been performed. For READ requests
  959. * we can let i_mutex go now that its achieved its purpose
  960. * of protecting us from looking up uninitialized blocks.
  961. */
  962. if ((rw == READ) && (dio->lock_type == DIO_LOCKING))
  963. mutex_unlock(&dio->inode->i_mutex);
  964. /*
  965. * The only time we want to leave bios in flight is when a successful
  966. * partial aio read or full aio write have been setup. In that case
  967. * bio completion will call aio_complete. The only time it's safe to
  968. * call aio_complete is when we return -EIOCBQUEUED, so we key on that.
  969. * This had *better* be the only place that raises -EIOCBQUEUED.
  970. */
  971. BUG_ON(ret == -EIOCBQUEUED);
  972. if (dio->is_async && ret == 0 && dio->result &&
  973. ((rw & READ) || (dio->result == dio->size)))
  974. ret = -EIOCBQUEUED;
  975. if (ret != -EIOCBQUEUED)
  976. dio_await_completion(dio);
  977. /*
  978. * Sync will always be dropping the final ref and completing the
  979. * operation. AIO can if it was a broken operation described above or
  980. * in fact if all the bios race to complete before we get here. In
  981. * that case dio_complete() translates the EIOCBQUEUED into the proper
  982. * return code that the caller will hand to aio_complete().
  983. *
  984. * This is managed by the bio_lock instead of being an atomic_t so that
  985. * completion paths can drop their ref and use the remaining count to
  986. * decide to wake the submission path atomically.
  987. */
  988. spin_lock_irqsave(&dio->bio_lock, flags);
  989. ret2 = --dio->refcount;
  990. spin_unlock_irqrestore(&dio->bio_lock, flags);
  991. BUG_ON(!dio->is_async && ret2 != 0);
  992. if (ret2 == 0) {
  993. ret = dio_complete(dio, offset, ret);
  994. kfree(dio);
  995. } else
  996. BUG_ON(ret != -EIOCBQUEUED);
  997. return ret;
  998. }
  999. /*
  1000. * This is a library function for use by filesystem drivers.
  1001. * The locking rules are governed by the dio_lock_type parameter.
  1002. *
  1003. * DIO_NO_LOCKING (no locking, for raw block device access)
  1004. * For writes, i_mutex is not held on entry; it is never taken.
  1005. *
  1006. * DIO_LOCKING (simple locking for regular files)
  1007. * For writes we are called under i_mutex and return with i_mutex held, even
  1008. * though it is internally dropped.
  1009. * For reads, i_mutex is not held on entry, but it is taken and dropped before
  1010. * returning.
  1011. *
  1012. * DIO_OWN_LOCKING (filesystem provides synchronisation and handling of
  1013. * uninitialised data, allowing parallel direct readers and writers)
  1014. * For writes we are called without i_mutex, return without it, never touch it.
  1015. * For reads we are called under i_mutex and return with i_mutex held, even
  1016. * though it may be internally dropped.
  1017. *
  1018. * Additional i_alloc_sem locking requirements described inline below.
  1019. */
  1020. ssize_t
  1021. __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
  1022. struct block_device *bdev, const struct iovec *iov, loff_t offset,
  1023. unsigned long nr_segs, get_block_t get_block, dio_iodone_t end_io,
  1024. int dio_lock_type)
  1025. {
  1026. int seg;
  1027. size_t size;
  1028. unsigned long addr;
  1029. unsigned blkbits = inode->i_blkbits;
  1030. unsigned bdev_blkbits = 0;
  1031. unsigned blocksize_mask = (1 << blkbits) - 1;
  1032. ssize_t retval = -EINVAL;
  1033. loff_t end = offset;
  1034. struct dio *dio;
  1035. int release_i_mutex = 0;
  1036. int acquire_i_mutex = 0;
  1037. if (rw & WRITE)
  1038. rw = WRITE_SYNC;
  1039. if (bdev)
  1040. bdev_blkbits = blksize_bits(bdev_hardsect_size(bdev));
  1041. if (offset & blocksize_mask) {
  1042. if (bdev)
  1043. blkbits = bdev_blkbits;
  1044. blocksize_mask = (1 << blkbits) - 1;
  1045. if (offset & blocksize_mask)
  1046. goto out;
  1047. }
  1048. /* Check the memory alignment. Blocks cannot straddle pages */
  1049. for (seg = 0; seg < nr_segs; seg++) {
  1050. addr = (unsigned long)iov[seg].iov_base;
  1051. size = iov[seg].iov_len;
  1052. end += size;
  1053. if ((addr & blocksize_mask) || (size & blocksize_mask)) {
  1054. if (bdev)
  1055. blkbits = bdev_blkbits;
  1056. blocksize_mask = (1 << blkbits) - 1;
  1057. if ((addr & blocksize_mask) || (size & blocksize_mask))
  1058. goto out;
  1059. }
  1060. }
  1061. dio = kmalloc(sizeof(*dio), GFP_KERNEL);
  1062. retval = -ENOMEM;
  1063. if (!dio)
  1064. goto out;
  1065. /*
  1066. * For block device access DIO_NO_LOCKING is used,
  1067. * neither readers nor writers do any locking at all
  1068. * For regular files using DIO_LOCKING,
  1069. * readers need to grab i_mutex and i_alloc_sem
  1070. * writers need to grab i_alloc_sem only (i_mutex is already held)
  1071. * For regular files using DIO_OWN_LOCKING,
  1072. * neither readers nor writers take any locks here
  1073. */
  1074. dio->lock_type = dio_lock_type;
  1075. if (dio_lock_type != DIO_NO_LOCKING) {
  1076. /* watch out for a 0 len io from a tricksy fs */
  1077. if (rw == READ && end > offset) {
  1078. struct address_space *mapping;
  1079. mapping = iocb->ki_filp->f_mapping;
  1080. if (dio_lock_type != DIO_OWN_LOCKING) {
  1081. mutex_lock(&inode->i_mutex);
  1082. release_i_mutex = 1;
  1083. }
  1084. retval = filemap_write_and_wait_range(mapping, offset,
  1085. end - 1);
  1086. if (retval) {
  1087. kfree(dio);
  1088. goto out;
  1089. }
  1090. if (dio_lock_type == DIO_OWN_LOCKING) {
  1091. mutex_unlock(&inode->i_mutex);
  1092. acquire_i_mutex = 1;
  1093. }
  1094. }
  1095. if (dio_lock_type == DIO_LOCKING)
  1096. /* lockdep: not the owner will release it */
  1097. down_read_non_owner(&inode->i_alloc_sem);
  1098. }
  1099. /*
  1100. * For file extending writes updating i_size before data
  1101. * writeouts complete can expose uninitialized blocks. So
  1102. * even for AIO, we need to wait for i/o to complete before
  1103. * returning in this case.
  1104. */
  1105. dio->is_async = !is_sync_kiocb(iocb) && !((rw & WRITE) &&
  1106. (end > i_size_read(inode)));
  1107. retval = direct_io_worker(rw, iocb, inode, iov, offset,
  1108. nr_segs, blkbits, get_block, end_io, dio);
  1109. if (rw == READ && dio_lock_type == DIO_LOCKING)
  1110. release_i_mutex = 0;
  1111. out:
  1112. if (release_i_mutex)
  1113. mutex_unlock(&inode->i_mutex);
  1114. else if (acquire_i_mutex)
  1115. mutex_lock(&inode->i_mutex);
  1116. return retval;
  1117. }
  1118. EXPORT_SYMBOL(__blockdev_direct_IO);