bio.c 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266
  1. /*
  2. * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License version 2 as
  6. * published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. * GNU General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public Licens
  14. * along with this program; if not, write to the Free Software
  15. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
  16. *
  17. */
  18. #include <linux/mm.h>
  19. #include <linux/swap.h>
  20. #include <linux/bio.h>
  21. #include <linux/blkdev.h>
  22. #include <linux/slab.h>
  23. #include <linux/init.h>
  24. #include <linux/kernel.h>
  25. #include <linux/module.h>
  26. #include <linux/mempool.h>
  27. #include <linux/workqueue.h>
  28. #include <linux/blktrace_api.h>
  29. #include <scsi/sg.h> /* for struct sg_iovec */
  30. #define BIO_POOL_SIZE 256
  31. static struct kmem_cache *bio_slab __read_mostly;
  32. #define BIOVEC_NR_POOLS 6
  33. /*
  34. * a small number of entries is fine, not going to be performance critical.
  35. * basically we just need to survive
  36. */
  37. #define BIO_SPLIT_ENTRIES 8
  38. mempool_t *bio_split_pool __read_mostly;
  39. struct biovec_slab {
  40. int nr_vecs;
  41. char *name;
  42. struct kmem_cache *slab;
  43. };
  44. /*
  45. * if you change this list, also change bvec_alloc or things will
  46. * break badly! cannot be bigger than what you can fit into an
  47. * unsigned short
  48. */
  49. #define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
  50. static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = {
  51. BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
  52. };
  53. #undef BV
  54. /*
  55. * bio_set is used to allow other portions of the IO system to
  56. * allocate their own private memory pools for bio and iovec structures.
  57. * These memory pools in turn all allocate from the bio_slab
  58. * and the bvec_slabs[].
  59. */
  60. struct bio_set {
  61. mempool_t *bio_pool;
  62. mempool_t *bvec_pools[BIOVEC_NR_POOLS];
  63. };
  64. /*
  65. * fs_bio_set is the bio_set containing bio and iovec memory pools used by
  66. * IO code that does not need private memory pools.
  67. */
  68. static struct bio_set *fs_bio_set;
  69. static inline struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx, struct bio_set *bs)
  70. {
  71. struct bio_vec *bvl;
  72. /*
  73. * see comment near bvec_array define!
  74. */
  75. switch (nr) {
  76. case 1 : *idx = 0; break;
  77. case 2 ... 4: *idx = 1; break;
  78. case 5 ... 16: *idx = 2; break;
  79. case 17 ... 64: *idx = 3; break;
  80. case 65 ... 128: *idx = 4; break;
  81. case 129 ... BIO_MAX_PAGES: *idx = 5; break;
  82. default:
  83. return NULL;
  84. }
  85. /*
  86. * idx now points to the pool we want to allocate from
  87. */
  88. bvl = mempool_alloc(bs->bvec_pools[*idx], gfp_mask);
  89. if (bvl) {
  90. struct biovec_slab *bp = bvec_slabs + *idx;
  91. memset(bvl, 0, bp->nr_vecs * sizeof(struct bio_vec));
  92. }
  93. return bvl;
  94. }
  95. void bio_free(struct bio *bio, struct bio_set *bio_set)
  96. {
  97. const int pool_idx = BIO_POOL_IDX(bio);
  98. BIO_BUG_ON(pool_idx >= BIOVEC_NR_POOLS);
  99. mempool_free(bio->bi_io_vec, bio_set->bvec_pools[pool_idx]);
  100. mempool_free(bio, bio_set->bio_pool);
  101. }
  102. /*
  103. * default destructor for a bio allocated with bio_alloc_bioset()
  104. */
  105. static void bio_fs_destructor(struct bio *bio)
  106. {
  107. bio_free(bio, fs_bio_set);
  108. }
  109. void bio_init(struct bio *bio)
  110. {
  111. bio->bi_next = NULL;
  112. bio->bi_bdev = NULL;
  113. bio->bi_flags = 1 << BIO_UPTODATE;
  114. bio->bi_rw = 0;
  115. bio->bi_vcnt = 0;
  116. bio->bi_idx = 0;
  117. bio->bi_phys_segments = 0;
  118. bio->bi_hw_segments = 0;
  119. bio->bi_hw_front_size = 0;
  120. bio->bi_hw_back_size = 0;
  121. bio->bi_size = 0;
  122. bio->bi_max_vecs = 0;
  123. bio->bi_end_io = NULL;
  124. atomic_set(&bio->bi_cnt, 1);
  125. bio->bi_private = NULL;
  126. }
  127. /**
  128. * bio_alloc_bioset - allocate a bio for I/O
  129. * @gfp_mask: the GFP_ mask given to the slab allocator
  130. * @nr_iovecs: number of iovecs to pre-allocate
  131. * @bs: the bio_set to allocate from
  132. *
  133. * Description:
  134. * bio_alloc_bioset will first try it's on mempool to satisfy the allocation.
  135. * If %__GFP_WAIT is set then we will block on the internal pool waiting
  136. * for a &struct bio to become free.
  137. *
  138. * allocate bio and iovecs from the memory pools specified by the
  139. * bio_set structure.
  140. **/
  141. struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
  142. {
  143. struct bio *bio = mempool_alloc(bs->bio_pool, gfp_mask);
  144. if (likely(bio)) {
  145. struct bio_vec *bvl = NULL;
  146. bio_init(bio);
  147. if (likely(nr_iovecs)) {
  148. unsigned long idx = 0; /* shut up gcc */
  149. bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);
  150. if (unlikely(!bvl)) {
  151. mempool_free(bio, bs->bio_pool);
  152. bio = NULL;
  153. goto out;
  154. }
  155. bio->bi_flags |= idx << BIO_POOL_OFFSET;
  156. bio->bi_max_vecs = bvec_slabs[idx].nr_vecs;
  157. }
  158. bio->bi_io_vec = bvl;
  159. }
  160. out:
  161. return bio;
  162. }
  163. struct bio *bio_alloc(gfp_t gfp_mask, int nr_iovecs)
  164. {
  165. struct bio *bio = bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set);
  166. if (bio)
  167. bio->bi_destructor = bio_fs_destructor;
  168. return bio;
  169. }
  170. void zero_fill_bio(struct bio *bio)
  171. {
  172. unsigned long flags;
  173. struct bio_vec *bv;
  174. int i;
  175. bio_for_each_segment(bv, bio, i) {
  176. char *data = bvec_kmap_irq(bv, &flags);
  177. memset(data, 0, bv->bv_len);
  178. flush_dcache_page(bv->bv_page);
  179. bvec_kunmap_irq(data, &flags);
  180. }
  181. }
  182. EXPORT_SYMBOL(zero_fill_bio);
  183. /**
  184. * bio_put - release a reference to a bio
  185. * @bio: bio to release reference to
  186. *
  187. * Description:
  188. * Put a reference to a &struct bio, either one you have gotten with
  189. * bio_alloc or bio_get. The last put of a bio will free it.
  190. **/
  191. void bio_put(struct bio *bio)
  192. {
  193. BIO_BUG_ON(!atomic_read(&bio->bi_cnt));
  194. /*
  195. * last put frees it
  196. */
  197. if (atomic_dec_and_test(&bio->bi_cnt)) {
  198. bio->bi_next = NULL;
  199. bio->bi_destructor(bio);
  200. }
  201. }
  202. inline int bio_phys_segments(request_queue_t *q, struct bio *bio)
  203. {
  204. if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
  205. blk_recount_segments(q, bio);
  206. return bio->bi_phys_segments;
  207. }
  208. inline int bio_hw_segments(request_queue_t *q, struct bio *bio)
  209. {
  210. if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
  211. blk_recount_segments(q, bio);
  212. return bio->bi_hw_segments;
  213. }
  214. /**
  215. * __bio_clone - clone a bio
  216. * @bio: destination bio
  217. * @bio_src: bio to clone
  218. *
  219. * Clone a &bio. Caller will own the returned bio, but not
  220. * the actual data it points to. Reference count of returned
  221. * bio will be one.
  222. */
  223. void __bio_clone(struct bio *bio, struct bio *bio_src)
  224. {
  225. request_queue_t *q = bdev_get_queue(bio_src->bi_bdev);
  226. memcpy(bio->bi_io_vec, bio_src->bi_io_vec,
  227. bio_src->bi_max_vecs * sizeof(struct bio_vec));
  228. bio->bi_sector = bio_src->bi_sector;
  229. bio->bi_bdev = bio_src->bi_bdev;
  230. bio->bi_flags |= 1 << BIO_CLONED;
  231. bio->bi_rw = bio_src->bi_rw;
  232. bio->bi_vcnt = bio_src->bi_vcnt;
  233. bio->bi_size = bio_src->bi_size;
  234. bio->bi_idx = bio_src->bi_idx;
  235. bio_phys_segments(q, bio);
  236. bio_hw_segments(q, bio);
  237. }
  238. /**
  239. * bio_clone - clone a bio
  240. * @bio: bio to clone
  241. * @gfp_mask: allocation priority
  242. *
  243. * Like __bio_clone, only also allocates the returned bio
  244. */
  245. struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask)
  246. {
  247. struct bio *b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, fs_bio_set);
  248. if (b) {
  249. b->bi_destructor = bio_fs_destructor;
  250. __bio_clone(b, bio);
  251. }
  252. return b;
  253. }
  254. /**
  255. * bio_get_nr_vecs - return approx number of vecs
  256. * @bdev: I/O target
  257. *
  258. * Return the approximate number of pages we can send to this target.
  259. * There's no guarantee that you will be able to fit this number of pages
  260. * into a bio, it does not account for dynamic restrictions that vary
  261. * on offset.
  262. */
  263. int bio_get_nr_vecs(struct block_device *bdev)
  264. {
  265. request_queue_t *q = bdev_get_queue(bdev);
  266. int nr_pages;
  267. nr_pages = ((q->max_sectors << 9) + PAGE_SIZE - 1) >> PAGE_SHIFT;
  268. if (nr_pages > q->max_phys_segments)
  269. nr_pages = q->max_phys_segments;
  270. if (nr_pages > q->max_hw_segments)
  271. nr_pages = q->max_hw_segments;
  272. return nr_pages;
  273. }
  274. static int __bio_add_page(request_queue_t *q, struct bio *bio, struct page
  275. *page, unsigned int len, unsigned int offset,
  276. unsigned short max_sectors)
  277. {
  278. int retried_segments = 0;
  279. struct bio_vec *bvec;
  280. /*
  281. * cloned bio must not modify vec list
  282. */
  283. if (unlikely(bio_flagged(bio, BIO_CLONED)))
  284. return 0;
  285. if (((bio->bi_size + len) >> 9) > max_sectors)
  286. return 0;
  287. /*
  288. * For filesystems with a blocksize smaller than the pagesize
  289. * we will often be called with the same page as last time and
  290. * a consecutive offset. Optimize this special case.
  291. */
  292. if (bio->bi_vcnt > 0) {
  293. struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
  294. if (page == prev->bv_page &&
  295. offset == prev->bv_offset + prev->bv_len) {
  296. prev->bv_len += len;
  297. if (q->merge_bvec_fn &&
  298. q->merge_bvec_fn(q, bio, prev) < len) {
  299. prev->bv_len -= len;
  300. return 0;
  301. }
  302. goto done;
  303. }
  304. }
  305. if (bio->bi_vcnt >= bio->bi_max_vecs)
  306. return 0;
  307. /*
  308. * we might lose a segment or two here, but rather that than
  309. * make this too complex.
  310. */
  311. while (bio->bi_phys_segments >= q->max_phys_segments
  312. || bio->bi_hw_segments >= q->max_hw_segments
  313. || BIOVEC_VIRT_OVERSIZE(bio->bi_size)) {
  314. if (retried_segments)
  315. return 0;
  316. retried_segments = 1;
  317. blk_recount_segments(q, bio);
  318. }
  319. /*
  320. * setup the new entry, we might clear it again later if we
  321. * cannot add the page
  322. */
  323. bvec = &bio->bi_io_vec[bio->bi_vcnt];
  324. bvec->bv_page = page;
  325. bvec->bv_len = len;
  326. bvec->bv_offset = offset;
  327. /*
  328. * if queue has other restrictions (eg varying max sector size
  329. * depending on offset), it can specify a merge_bvec_fn in the
  330. * queue to get further control
  331. */
  332. if (q->merge_bvec_fn) {
  333. /*
  334. * merge_bvec_fn() returns number of bytes it can accept
  335. * at this offset
  336. */
  337. if (q->merge_bvec_fn(q, bio, bvec) < len) {
  338. bvec->bv_page = NULL;
  339. bvec->bv_len = 0;
  340. bvec->bv_offset = 0;
  341. return 0;
  342. }
  343. }
  344. /* If we may be able to merge these biovecs, force a recount */
  345. if (bio->bi_vcnt && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec) ||
  346. BIOVEC_VIRT_MERGEABLE(bvec-1, bvec)))
  347. bio->bi_flags &= ~(1 << BIO_SEG_VALID);
  348. bio->bi_vcnt++;
  349. bio->bi_phys_segments++;
  350. bio->bi_hw_segments++;
  351. done:
  352. bio->bi_size += len;
  353. return len;
  354. }
  355. /**
  356. * bio_add_pc_page - attempt to add page to bio
  357. * @q: the target queue
  358. * @bio: destination bio
  359. * @page: page to add
  360. * @len: vec entry length
  361. * @offset: vec entry offset
  362. *
  363. * Attempt to add a page to the bio_vec maplist. This can fail for a
  364. * number of reasons, such as the bio being full or target block
  365. * device limitations. The target block device must allow bio's
  366. * smaller than PAGE_SIZE, so it is always possible to add a single
  367. * page to an empty bio. This should only be used by REQ_PC bios.
  368. */
  369. int bio_add_pc_page(request_queue_t *q, struct bio *bio, struct page *page,
  370. unsigned int len, unsigned int offset)
  371. {
  372. return __bio_add_page(q, bio, page, len, offset, q->max_hw_sectors);
  373. }
  374. /**
  375. * bio_add_page - attempt to add page to bio
  376. * @bio: destination bio
  377. * @page: page to add
  378. * @len: vec entry length
  379. * @offset: vec entry offset
  380. *
  381. * Attempt to add a page to the bio_vec maplist. This can fail for a
  382. * number of reasons, such as the bio being full or target block
  383. * device limitations. The target block device must allow bio's
  384. * smaller than PAGE_SIZE, so it is always possible to add a single
  385. * page to an empty bio.
  386. */
  387. int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
  388. unsigned int offset)
  389. {
  390. struct request_queue *q = bdev_get_queue(bio->bi_bdev);
  391. return __bio_add_page(q, bio, page, len, offset, q->max_sectors);
  392. }
  393. struct bio_map_data {
  394. struct bio_vec *iovecs;
  395. void __user *userptr;
  396. };
  397. static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio)
  398. {
  399. memcpy(bmd->iovecs, bio->bi_io_vec, sizeof(struct bio_vec) * bio->bi_vcnt);
  400. bio->bi_private = bmd;
  401. }
  402. static void bio_free_map_data(struct bio_map_data *bmd)
  403. {
  404. kfree(bmd->iovecs);
  405. kfree(bmd);
  406. }
  407. static struct bio_map_data *bio_alloc_map_data(int nr_segs)
  408. {
  409. struct bio_map_data *bmd = kmalloc(sizeof(*bmd), GFP_KERNEL);
  410. if (!bmd)
  411. return NULL;
  412. bmd->iovecs = kmalloc(sizeof(struct bio_vec) * nr_segs, GFP_KERNEL);
  413. if (bmd->iovecs)
  414. return bmd;
  415. kfree(bmd);
  416. return NULL;
  417. }
  418. /**
  419. * bio_uncopy_user - finish previously mapped bio
  420. * @bio: bio being terminated
  421. *
  422. * Free pages allocated from bio_copy_user() and write back data
  423. * to user space in case of a read.
  424. */
  425. int bio_uncopy_user(struct bio *bio)
  426. {
  427. struct bio_map_data *bmd = bio->bi_private;
  428. const int read = bio_data_dir(bio) == READ;
  429. struct bio_vec *bvec;
  430. int i, ret = 0;
  431. __bio_for_each_segment(bvec, bio, i, 0) {
  432. char *addr = page_address(bvec->bv_page);
  433. unsigned int len = bmd->iovecs[i].bv_len;
  434. if (read && !ret && copy_to_user(bmd->userptr, addr, len))
  435. ret = -EFAULT;
  436. __free_page(bvec->bv_page);
  437. bmd->userptr += len;
  438. }
  439. bio_free_map_data(bmd);
  440. bio_put(bio);
  441. return ret;
  442. }
  443. /**
  444. * bio_copy_user - copy user data to bio
  445. * @q: destination block queue
  446. * @uaddr: start of user address
  447. * @len: length in bytes
  448. * @write_to_vm: bool indicating writing to pages or not
  449. *
  450. * Prepares and returns a bio for indirect user io, bouncing data
  451. * to/from kernel pages as necessary. Must be paired with
  452. * call bio_uncopy_user() on io completion.
  453. */
  454. struct bio *bio_copy_user(request_queue_t *q, unsigned long uaddr,
  455. unsigned int len, int write_to_vm)
  456. {
  457. unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  458. unsigned long start = uaddr >> PAGE_SHIFT;
  459. struct bio_map_data *bmd;
  460. struct bio_vec *bvec;
  461. struct page *page;
  462. struct bio *bio;
  463. int i, ret;
  464. bmd = bio_alloc_map_data(end - start);
  465. if (!bmd)
  466. return ERR_PTR(-ENOMEM);
  467. bmd->userptr = (void __user *) uaddr;
  468. ret = -ENOMEM;
  469. bio = bio_alloc(GFP_KERNEL, end - start);
  470. if (!bio)
  471. goto out_bmd;
  472. bio->bi_rw |= (!write_to_vm << BIO_RW);
  473. ret = 0;
  474. while (len) {
  475. unsigned int bytes = PAGE_SIZE;
  476. if (bytes > len)
  477. bytes = len;
  478. page = alloc_page(q->bounce_gfp | GFP_KERNEL);
  479. if (!page) {
  480. ret = -ENOMEM;
  481. break;
  482. }
  483. if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes)
  484. break;
  485. len -= bytes;
  486. }
  487. if (ret)
  488. goto cleanup;
  489. /*
  490. * success
  491. */
  492. if (!write_to_vm) {
  493. char __user *p = (char __user *) uaddr;
  494. /*
  495. * for a write, copy in data to kernel pages
  496. */
  497. ret = -EFAULT;
  498. bio_for_each_segment(bvec, bio, i) {
  499. char *addr = page_address(bvec->bv_page);
  500. if (copy_from_user(addr, p, bvec->bv_len))
  501. goto cleanup;
  502. p += bvec->bv_len;
  503. }
  504. }
  505. bio_set_map_data(bmd, bio);
  506. return bio;
  507. cleanup:
  508. bio_for_each_segment(bvec, bio, i)
  509. __free_page(bvec->bv_page);
  510. bio_put(bio);
  511. out_bmd:
  512. bio_free_map_data(bmd);
  513. return ERR_PTR(ret);
  514. }
  515. static struct bio *__bio_map_user_iov(request_queue_t *q,
  516. struct block_device *bdev,
  517. struct sg_iovec *iov, int iov_count,
  518. int write_to_vm)
  519. {
  520. int i, j;
  521. int nr_pages = 0;
  522. struct page **pages;
  523. struct bio *bio;
  524. int cur_page = 0;
  525. int ret, offset;
  526. for (i = 0; i < iov_count; i++) {
  527. unsigned long uaddr = (unsigned long)iov[i].iov_base;
  528. unsigned long len = iov[i].iov_len;
  529. unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  530. unsigned long start = uaddr >> PAGE_SHIFT;
  531. nr_pages += end - start;
  532. /*
  533. * buffer must be aligned to at least hardsector size for now
  534. */
  535. if (uaddr & queue_dma_alignment(q))
  536. return ERR_PTR(-EINVAL);
  537. }
  538. if (!nr_pages)
  539. return ERR_PTR(-EINVAL);
  540. bio = bio_alloc(GFP_KERNEL, nr_pages);
  541. if (!bio)
  542. return ERR_PTR(-ENOMEM);
  543. ret = -ENOMEM;
  544. pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
  545. if (!pages)
  546. goto out;
  547. for (i = 0; i < iov_count; i++) {
  548. unsigned long uaddr = (unsigned long)iov[i].iov_base;
  549. unsigned long len = iov[i].iov_len;
  550. unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  551. unsigned long start = uaddr >> PAGE_SHIFT;
  552. const int local_nr_pages = end - start;
  553. const int page_limit = cur_page + local_nr_pages;
  554. down_read(&current->mm->mmap_sem);
  555. ret = get_user_pages(current, current->mm, uaddr,
  556. local_nr_pages,
  557. write_to_vm, 0, &pages[cur_page], NULL);
  558. up_read(&current->mm->mmap_sem);
  559. if (ret < local_nr_pages) {
  560. ret = -EFAULT;
  561. goto out_unmap;
  562. }
  563. offset = uaddr & ~PAGE_MASK;
  564. for (j = cur_page; j < page_limit; j++) {
  565. unsigned int bytes = PAGE_SIZE - offset;
  566. if (len <= 0)
  567. break;
  568. if (bytes > len)
  569. bytes = len;
  570. /*
  571. * sorry...
  572. */
  573. if (bio_add_pc_page(q, bio, pages[j], bytes, offset) <
  574. bytes)
  575. break;
  576. len -= bytes;
  577. offset = 0;
  578. }
  579. cur_page = j;
  580. /*
  581. * release the pages we didn't map into the bio, if any
  582. */
  583. while (j < page_limit)
  584. page_cache_release(pages[j++]);
  585. }
  586. kfree(pages);
  587. /*
  588. * set data direction, and check if mapped pages need bouncing
  589. */
  590. if (!write_to_vm)
  591. bio->bi_rw |= (1 << BIO_RW);
  592. bio->bi_bdev = bdev;
  593. bio->bi_flags |= (1 << BIO_USER_MAPPED);
  594. return bio;
  595. out_unmap:
  596. for (i = 0; i < nr_pages; i++) {
  597. if(!pages[i])
  598. break;
  599. page_cache_release(pages[i]);
  600. }
  601. out:
  602. kfree(pages);
  603. bio_put(bio);
  604. return ERR_PTR(ret);
  605. }
  606. /**
  607. * bio_map_user - map user address into bio
  608. * @q: the request_queue_t for the bio
  609. * @bdev: destination block device
  610. * @uaddr: start of user address
  611. * @len: length in bytes
  612. * @write_to_vm: bool indicating writing to pages or not
  613. *
  614. * Map the user space address into a bio suitable for io to a block
  615. * device. Returns an error pointer in case of error.
  616. */
  617. struct bio *bio_map_user(request_queue_t *q, struct block_device *bdev,
  618. unsigned long uaddr, unsigned int len, int write_to_vm)
  619. {
  620. struct sg_iovec iov;
  621. iov.iov_base = (void __user *)uaddr;
  622. iov.iov_len = len;
  623. return bio_map_user_iov(q, bdev, &iov, 1, write_to_vm);
  624. }
  625. /**
  626. * bio_map_user_iov - map user sg_iovec table into bio
  627. * @q: the request_queue_t for the bio
  628. * @bdev: destination block device
  629. * @iov: the iovec.
  630. * @iov_count: number of elements in the iovec
  631. * @write_to_vm: bool indicating writing to pages or not
  632. *
  633. * Map the user space address into a bio suitable for io to a block
  634. * device. Returns an error pointer in case of error.
  635. */
  636. struct bio *bio_map_user_iov(request_queue_t *q, struct block_device *bdev,
  637. struct sg_iovec *iov, int iov_count,
  638. int write_to_vm)
  639. {
  640. struct bio *bio;
  641. bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm);
  642. if (IS_ERR(bio))
  643. return bio;
  644. /*
  645. * subtle -- if __bio_map_user() ended up bouncing a bio,
  646. * it would normally disappear when its bi_end_io is run.
  647. * however, we need it for the unmap, so grab an extra
  648. * reference to it
  649. */
  650. bio_get(bio);
  651. return bio;
  652. }
  653. static void __bio_unmap_user(struct bio *bio)
  654. {
  655. struct bio_vec *bvec;
  656. int i;
  657. /*
  658. * make sure we dirty pages we wrote to
  659. */
  660. __bio_for_each_segment(bvec, bio, i, 0) {
  661. if (bio_data_dir(bio) == READ)
  662. set_page_dirty_lock(bvec->bv_page);
  663. page_cache_release(bvec->bv_page);
  664. }
  665. bio_put(bio);
  666. }
  667. /**
  668. * bio_unmap_user - unmap a bio
  669. * @bio: the bio being unmapped
  670. *
  671. * Unmap a bio previously mapped by bio_map_user(). Must be called with
  672. * a process context.
  673. *
  674. * bio_unmap_user() may sleep.
  675. */
  676. void bio_unmap_user(struct bio *bio)
  677. {
  678. __bio_unmap_user(bio);
  679. bio_put(bio);
  680. }
  681. static int bio_map_kern_endio(struct bio *bio, unsigned int bytes_done, int err)
  682. {
  683. if (bio->bi_size)
  684. return 1;
  685. bio_put(bio);
  686. return 0;
  687. }
  688. static struct bio *__bio_map_kern(request_queue_t *q, void *data,
  689. unsigned int len, gfp_t gfp_mask)
  690. {
  691. unsigned long kaddr = (unsigned long)data;
  692. unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  693. unsigned long start = kaddr >> PAGE_SHIFT;
  694. const int nr_pages = end - start;
  695. int offset, i;
  696. struct bio *bio;
  697. bio = bio_alloc(gfp_mask, nr_pages);
  698. if (!bio)
  699. return ERR_PTR(-ENOMEM);
  700. offset = offset_in_page(kaddr);
  701. for (i = 0; i < nr_pages; i++) {
  702. unsigned int bytes = PAGE_SIZE - offset;
  703. if (len <= 0)
  704. break;
  705. if (bytes > len)
  706. bytes = len;
  707. if (bio_add_pc_page(q, bio, virt_to_page(data), bytes,
  708. offset) < bytes)
  709. break;
  710. data += bytes;
  711. len -= bytes;
  712. offset = 0;
  713. }
  714. bio->bi_end_io = bio_map_kern_endio;
  715. return bio;
  716. }
  717. /**
  718. * bio_map_kern - map kernel address into bio
  719. * @q: the request_queue_t for the bio
  720. * @data: pointer to buffer to map
  721. * @len: length in bytes
  722. * @gfp_mask: allocation flags for bio allocation
  723. *
  724. * Map the kernel address into a bio suitable for io to a block
  725. * device. Returns an error pointer in case of error.
  726. */
  727. struct bio *bio_map_kern(request_queue_t *q, void *data, unsigned int len,
  728. gfp_t gfp_mask)
  729. {
  730. struct bio *bio;
  731. bio = __bio_map_kern(q, data, len, gfp_mask);
  732. if (IS_ERR(bio))
  733. return bio;
  734. if (bio->bi_size == len)
  735. return bio;
  736. /*
  737. * Don't support partial mappings.
  738. */
  739. bio_put(bio);
  740. return ERR_PTR(-EINVAL);
  741. }
  742. /*
  743. * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
  744. * for performing direct-IO in BIOs.
  745. *
  746. * The problem is that we cannot run set_page_dirty() from interrupt context
  747. * because the required locks are not interrupt-safe. So what we can do is to
  748. * mark the pages dirty _before_ performing IO. And in interrupt context,
  749. * check that the pages are still dirty. If so, fine. If not, redirty them
  750. * in process context.
  751. *
  752. * We special-case compound pages here: normally this means reads into hugetlb
  753. * pages. The logic in here doesn't really work right for compound pages
  754. * because the VM does not uniformly chase down the head page in all cases.
  755. * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
  756. * handle them at all. So we skip compound pages here at an early stage.
  757. *
  758. * Note that this code is very hard to test under normal circumstances because
  759. * direct-io pins the pages with get_user_pages(). This makes
  760. * is_page_cache_freeable return false, and the VM will not clean the pages.
  761. * But other code (eg, pdflush) could clean the pages if they are mapped
  762. * pagecache.
  763. *
  764. * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
  765. * deferred bio dirtying paths.
  766. */
  767. /*
  768. * bio_set_pages_dirty() will mark all the bio's pages as dirty.
  769. */
  770. void bio_set_pages_dirty(struct bio *bio)
  771. {
  772. struct bio_vec *bvec = bio->bi_io_vec;
  773. int i;
  774. for (i = 0; i < bio->bi_vcnt; i++) {
  775. struct page *page = bvec[i].bv_page;
  776. if (page && !PageCompound(page))
  777. set_page_dirty_lock(page);
  778. }
  779. }
  780. void bio_release_pages(struct bio *bio)
  781. {
  782. struct bio_vec *bvec = bio->bi_io_vec;
  783. int i;
  784. for (i = 0; i < bio->bi_vcnt; i++) {
  785. struct page *page = bvec[i].bv_page;
  786. if (page)
  787. put_page(page);
  788. }
  789. }
  790. /*
  791. * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
  792. * If they are, then fine. If, however, some pages are clean then they must
  793. * have been written out during the direct-IO read. So we take another ref on
  794. * the BIO and the offending pages and re-dirty the pages in process context.
  795. *
  796. * It is expected that bio_check_pages_dirty() will wholly own the BIO from
  797. * here on. It will run one page_cache_release() against each page and will
  798. * run one bio_put() against the BIO.
  799. */
  800. static void bio_dirty_fn(struct work_struct *work);
  801. static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
  802. static DEFINE_SPINLOCK(bio_dirty_lock);
  803. static struct bio *bio_dirty_list;
  804. /*
  805. * This runs in process context
  806. */
  807. static void bio_dirty_fn(struct work_struct *work)
  808. {
  809. unsigned long flags;
  810. struct bio *bio;
  811. spin_lock_irqsave(&bio_dirty_lock, flags);
  812. bio = bio_dirty_list;
  813. bio_dirty_list = NULL;
  814. spin_unlock_irqrestore(&bio_dirty_lock, flags);
  815. while (bio) {
  816. struct bio *next = bio->bi_private;
  817. bio_set_pages_dirty(bio);
  818. bio_release_pages(bio);
  819. bio_put(bio);
  820. bio = next;
  821. }
  822. }
  823. void bio_check_pages_dirty(struct bio *bio)
  824. {
  825. struct bio_vec *bvec = bio->bi_io_vec;
  826. int nr_clean_pages = 0;
  827. int i;
  828. for (i = 0; i < bio->bi_vcnt; i++) {
  829. struct page *page = bvec[i].bv_page;
  830. if (PageDirty(page) || PageCompound(page)) {
  831. page_cache_release(page);
  832. bvec[i].bv_page = NULL;
  833. } else {
  834. nr_clean_pages++;
  835. }
  836. }
  837. if (nr_clean_pages) {
  838. unsigned long flags;
  839. spin_lock_irqsave(&bio_dirty_lock, flags);
  840. bio->bi_private = bio_dirty_list;
  841. bio_dirty_list = bio;
  842. spin_unlock_irqrestore(&bio_dirty_lock, flags);
  843. schedule_work(&bio_dirty_work);
  844. } else {
  845. bio_put(bio);
  846. }
  847. }
  848. /**
  849. * bio_endio - end I/O on a bio
  850. * @bio: bio
  851. * @bytes_done: number of bytes completed
  852. * @error: error, if any
  853. *
  854. * Description:
  855. * bio_endio() will end I/O on @bytes_done number of bytes. This may be
  856. * just a partial part of the bio, or it may be the whole bio. bio_endio()
  857. * is the preferred way to end I/O on a bio, it takes care of decrementing
  858. * bi_size and clearing BIO_UPTODATE on error. @error is 0 on success, and
  859. * and one of the established -Exxxx (-EIO, for instance) error values in
  860. * case something went wrong. Noone should call bi_end_io() directly on
  861. * a bio unless they own it and thus know that it has an end_io function.
  862. **/
  863. void bio_endio(struct bio *bio, unsigned int bytes_done, int error)
  864. {
  865. if (error)
  866. clear_bit(BIO_UPTODATE, &bio->bi_flags);
  867. if (unlikely(bytes_done > bio->bi_size)) {
  868. printk("%s: want %u bytes done, only %u left\n", __FUNCTION__,
  869. bytes_done, bio->bi_size);
  870. bytes_done = bio->bi_size;
  871. }
  872. bio->bi_size -= bytes_done;
  873. bio->bi_sector += (bytes_done >> 9);
  874. if (bio->bi_end_io)
  875. bio->bi_end_io(bio, bytes_done, error);
  876. }
  877. void bio_pair_release(struct bio_pair *bp)
  878. {
  879. if (atomic_dec_and_test(&bp->cnt)) {
  880. struct bio *master = bp->bio1.bi_private;
  881. bio_endio(master, master->bi_size, bp->error);
  882. mempool_free(bp, bp->bio2.bi_private);
  883. }
  884. }
  885. static int bio_pair_end_1(struct bio * bi, unsigned int done, int err)
  886. {
  887. struct bio_pair *bp = container_of(bi, struct bio_pair, bio1);
  888. if (err)
  889. bp->error = err;
  890. if (bi->bi_size)
  891. return 1;
  892. bio_pair_release(bp);
  893. return 0;
  894. }
  895. static int bio_pair_end_2(struct bio * bi, unsigned int done, int err)
  896. {
  897. struct bio_pair *bp = container_of(bi, struct bio_pair, bio2);
  898. if (err)
  899. bp->error = err;
  900. if (bi->bi_size)
  901. return 1;
  902. bio_pair_release(bp);
  903. return 0;
  904. }
  905. /*
  906. * split a bio - only worry about a bio with a single page
  907. * in it's iovec
  908. */
  909. struct bio_pair *bio_split(struct bio *bi, mempool_t *pool, int first_sectors)
  910. {
  911. struct bio_pair *bp = mempool_alloc(pool, GFP_NOIO);
  912. if (!bp)
  913. return bp;
  914. blk_add_trace_pdu_int(bdev_get_queue(bi->bi_bdev), BLK_TA_SPLIT, bi,
  915. bi->bi_sector + first_sectors);
  916. BUG_ON(bi->bi_vcnt != 1);
  917. BUG_ON(bi->bi_idx != 0);
  918. atomic_set(&bp->cnt, 3);
  919. bp->error = 0;
  920. bp->bio1 = *bi;
  921. bp->bio2 = *bi;
  922. bp->bio2.bi_sector += first_sectors;
  923. bp->bio2.bi_size -= first_sectors << 9;
  924. bp->bio1.bi_size = first_sectors << 9;
  925. bp->bv1 = bi->bi_io_vec[0];
  926. bp->bv2 = bi->bi_io_vec[0];
  927. bp->bv2.bv_offset += first_sectors << 9;
  928. bp->bv2.bv_len -= first_sectors << 9;
  929. bp->bv1.bv_len = first_sectors << 9;
  930. bp->bio1.bi_io_vec = &bp->bv1;
  931. bp->bio2.bi_io_vec = &bp->bv2;
  932. bp->bio1.bi_max_vecs = 1;
  933. bp->bio2.bi_max_vecs = 1;
  934. bp->bio1.bi_end_io = bio_pair_end_1;
  935. bp->bio2.bi_end_io = bio_pair_end_2;
  936. bp->bio1.bi_private = bi;
  937. bp->bio2.bi_private = pool;
  938. return bp;
  939. }
  940. /*
  941. * create memory pools for biovec's in a bio_set.
  942. * use the global biovec slabs created for general use.
  943. */
  944. static int biovec_create_pools(struct bio_set *bs, int pool_entries, int scale)
  945. {
  946. int i;
  947. for (i = 0; i < BIOVEC_NR_POOLS; i++) {
  948. struct biovec_slab *bp = bvec_slabs + i;
  949. mempool_t **bvp = bs->bvec_pools + i;
  950. if (pool_entries > 1 && i >= scale)
  951. pool_entries >>= 1;
  952. *bvp = mempool_create_slab_pool(pool_entries, bp->slab);
  953. if (!*bvp)
  954. return -ENOMEM;
  955. }
  956. return 0;
  957. }
  958. static void biovec_free_pools(struct bio_set *bs)
  959. {
  960. int i;
  961. for (i = 0; i < BIOVEC_NR_POOLS; i++) {
  962. mempool_t *bvp = bs->bvec_pools[i];
  963. if (bvp)
  964. mempool_destroy(bvp);
  965. }
  966. }
  967. void bioset_free(struct bio_set *bs)
  968. {
  969. if (bs->bio_pool)
  970. mempool_destroy(bs->bio_pool);
  971. biovec_free_pools(bs);
  972. kfree(bs);
  973. }
  974. struct bio_set *bioset_create(int bio_pool_size, int bvec_pool_size, int scale)
  975. {
  976. struct bio_set *bs = kzalloc(sizeof(*bs), GFP_KERNEL);
  977. if (!bs)
  978. return NULL;
  979. bs->bio_pool = mempool_create_slab_pool(bio_pool_size, bio_slab);
  980. if (!bs->bio_pool)
  981. goto bad;
  982. if (!biovec_create_pools(bs, bvec_pool_size, scale))
  983. return bs;
  984. bad:
  985. bioset_free(bs);
  986. return NULL;
  987. }
  988. static void __init biovec_init_slabs(void)
  989. {
  990. int i;
  991. for (i = 0; i < BIOVEC_NR_POOLS; i++) {
  992. int size;
  993. struct biovec_slab *bvs = bvec_slabs + i;
  994. size = bvs->nr_vecs * sizeof(struct bio_vec);
  995. bvs->slab = kmem_cache_create(bvs->name, size, 0,
  996. SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
  997. }
  998. }
  999. static int __init init_bio(void)
  1000. {
  1001. int megabytes, bvec_pool_entries;
  1002. int scale = BIOVEC_NR_POOLS;
  1003. bio_slab = kmem_cache_create("bio", sizeof(struct bio), 0,
  1004. SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL, NULL);
  1005. biovec_init_slabs();
  1006. megabytes = nr_free_pages() >> (20 - PAGE_SHIFT);
  1007. /*
  1008. * find out where to start scaling
  1009. */
  1010. if (megabytes <= 16)
  1011. scale = 0;
  1012. else if (megabytes <= 32)
  1013. scale = 1;
  1014. else if (megabytes <= 64)
  1015. scale = 2;
  1016. else if (megabytes <= 96)
  1017. scale = 3;
  1018. else if (megabytes <= 128)
  1019. scale = 4;
  1020. /*
  1021. * Limit number of entries reserved -- mempools are only used when
  1022. * the system is completely unable to allocate memory, so we only
  1023. * need enough to make progress.
  1024. */
  1025. bvec_pool_entries = 1 + scale;
  1026. fs_bio_set = bioset_create(BIO_POOL_SIZE, bvec_pool_entries, scale);
  1027. if (!fs_bio_set)
  1028. panic("bio: can't allocate bios\n");
  1029. bio_split_pool = mempool_create_kmalloc_pool(BIO_SPLIT_ENTRIES,
  1030. sizeof(struct bio_pair));
  1031. if (!bio_split_pool)
  1032. panic("bio: can't create split pool\n");
  1033. return 0;
  1034. }
  1035. subsys_initcall(init_bio);
  1036. EXPORT_SYMBOL(bio_alloc);
  1037. EXPORT_SYMBOL(bio_put);
  1038. EXPORT_SYMBOL(bio_free);
  1039. EXPORT_SYMBOL(bio_endio);
  1040. EXPORT_SYMBOL(bio_init);
  1041. EXPORT_SYMBOL(__bio_clone);
  1042. EXPORT_SYMBOL(bio_clone);
  1043. EXPORT_SYMBOL(bio_phys_segments);
  1044. EXPORT_SYMBOL(bio_hw_segments);
  1045. EXPORT_SYMBOL(bio_add_page);
  1046. EXPORT_SYMBOL(bio_add_pc_page);
  1047. EXPORT_SYMBOL(bio_get_nr_vecs);
  1048. EXPORT_SYMBOL(bio_map_user);
  1049. EXPORT_SYMBOL(bio_unmap_user);
  1050. EXPORT_SYMBOL(bio_map_kern);
  1051. EXPORT_SYMBOL(bio_pair_release);
  1052. EXPORT_SYMBOL(bio_split);
  1053. EXPORT_SYMBOL(bio_split_pool);
  1054. EXPORT_SYMBOL(bio_copy_user);
  1055. EXPORT_SYMBOL(bio_uncopy_user);
  1056. EXPORT_SYMBOL(bioset_create);
  1057. EXPORT_SYMBOL(bioset_free);
  1058. EXPORT_SYMBOL(bio_alloc_bioset);